diff --git a/shortsmith/deployment.yaml b/shortsmith/deployment.yaml index b1013d6..d5e7242 100644 --- a/shortsmith/deployment.yaml +++ b/shortsmith/deployment.yaml @@ -56,33 +56,38 @@ spec: volumeMounts: - name: data mountPath: /data - # Measured on chemavx-k8 under `systemd-run --scope -p CPUQuota=300%`, - # sampling the cgroup's memory.stat anon every 20 ms — the figure that - # cannot be reclaimed. Rendering is this service's normal mode, not a - # burst, so the request comes from the peak and not from the resting 22 MB. + # Measured IN THIS POD, sampling the cgroup's own memory.stat every 30 ms. + # An earlier set of numbers taken on the dev host said 522 MB and produced a + # 1Gi limit; the first real render was OOMKilled at 1483 MB. The host runs + # ffmpeg 4.4.2 and the image ships 7.1.5, and 7.0 rewrote transcoding onto a + # threaded scheduler whose inter-component queues hold decoded frames. So: + # numbers measured outside the deployment target are a guess about it. # - # at rest (imports + FastAPI app) 22 MB - # frames, 3 workers, 1260 of them 147 MB - # audio synthesis and WAV 71 MB - # encode, -threads 3 480 MB <- the whole cost - # full render of examples/jal1628.json 522 MB, 32 s - # the same at the spec's 180 s ceiling 511 MB, 138 s + # Rendering is this service's normal mode, not a burst, so the request comes + # from the peak and not from the resting figure. # - # The peak is the encode's and does not grow with duration: at 5400 frames - # anon is unchanged and only the page cache grows (64 -> 275 MB), which is - # reclaimable. That is why these are sized against anon and not against - # memory.current, which reaches 808 MB at the ceiling. + # at rest (imports + FastAPI app) 40 MB current + # frames, 3 workers, 1260 of them +145 MB anon + # audio synthesis and WAV +82 MB anon + # encode pass 1, video only, -threads 3 +562 MB anon <- the cost + # encode pass 2, mux the audio in +15 MB anon + # two renders back to back, peak 640 MB anon / 716 current + # one render at the spec's 180 s ceiling 620 MB anon / 923 current # - # 512Mi does survive the reference render, but only by evicting page cache - # to sit exactly on the cap with no margin. 640Mi covers the measured peak - # with ~20% headroom; the 1Gi limit lets the page cache stay cached instead - # of being re-read during the encode. + # anon is flat with duration; what grows is the page cache holding the PNG + # frames (70 MB at 42 s, 287 MB at the 180 s ceiling). That is reclaimable, + # so the request is sized against anon and the limit against current. + # + # 768Mi covers the 640 MB anon peak with ~20% headroom. The limit is 1.5Gi + # and not 1Gi because of the ceiling case: at 1Gi that render peaked at + # 940 MB of 1024, i.e. 84 MB of margin, surviving only because the kernel + # had reclaimable cache to drop. Verified at 1.5Gi with memory.events + # reading `max 0` — the limit was never reached, not merely never fatal. # # The CPU limit is what sizes the worker pool, not the request — the code # reads cpu.max, since cpu_count() reports the node's 16. At limits.cpu 3 - # that is 3 workers, verified in a real cgroup (300%->3, 200%->2, 100%->1). - # 3 CPUs render in 32 s against 48 s at 2, for 22 MB less, so the third is - # worth more than the extra worker costs. + # that is 3 workers, verified in this pod. 42 s of video renders in ~31 s, + # the 180 s ceiling in ~135 s. # # Consequence, deliberate: under sustained contention the pod runs 3 workers # on the 1 CPU it is guaranteed and a render takes about 3x longer. That is @@ -94,10 +99,10 @@ spec: resources: requests: cpu: "1" - memory: "640Mi" + memory: "768Mi" limits: cpu: "3" - memory: "1Gi" + memory: "1536Mi" # The render runs off the event loop (asyncio.to_thread, and the frame # workers are separate processes), so /healthz answers while a job is in # flight. A probe timing out means the process is actually wedged.