1--- a/api/container_stats.go
2+++ b/api/container_stats.go
3@@ -10,6 +10,8 @@
4 "fmt"
5 "io"
6 "net/http"
7+ "net/url"
8+ "strings"
9 )
10
11 var ContainerNotFound = errors.New("No such Container")
12@@ -19,44 +21,52 @@
13 func (c *API) ContainerStats(ctx context.Context, name string) (Stats, error) {
14
15 var stats Stats
16- res, err := c.Get(ctx, fmt.Sprintf("/v1.0.0/libpod/containers/%s/stats?stream=false", name))
17+ res, err := c.Get(ctx, "/v4.0.0/libpod/containers/stats?stream=false&containers="+url.QueryEscape(name))
18 if err != nil {
19 return stats, err
20 }
21
22 defer ignoreClose(res.Body)
23
24- if res.StatusCode == http.StatusNotFound {
25- return stats, ContainerNotFound
26- }
27-
28- if res.StatusCode == http.StatusConflict {
29- return stats, ContainerWrongState
30- }
31- if res.StatusCode != http.StatusOK {
32- return stats, fmt.Errorf("cannot get stats of container, status code: %d", res.StatusCode)
33- }
34-
35 body, err := io.ReadAll(res.Body)
36 if err != nil {
37 return stats, err
38 }
39
40- // Since podman 4.1.1, an empty 200 response is returned for stopped containers.
41- if len(body) == 0 {
42- return stats, ContainerNotFound
43+ if res.StatusCode != http.StatusOK {
44+ var failure Error
45+ _ = json.Unmarshal(body, &failure)
46+ if res.StatusCode == http.StatusNotFound && strings.Contains(failure.Cause, "no such container") {
47+ return stats, ContainerNotFound
48+ }
49+ if strings.Contains(failure.Cause, "container is stopped") || strings.Contains(failure.Cause, "container state improper") {
50+ return stats, ContainerWrongState
51+ }
52+ return stats, fmt.Errorf("cannot get stats of container, status code: %d, cause: %s", res.StatusCode, failure.Cause)
53 }
54
55- // Since podman 4.6.0, a 200 response with `container is stopped` is returned for stopped containers.
56- var errResponse Error
57- if _ = json.Unmarshal(body, &errResponse); errResponse.Cause == "container is stopped" {
58- return stats, ContainerNotFound
59+ var report struct {
60+ Stats []struct {
61+ CPUNano, CPUSystemNano, SystemNano, MemUsage, MemLimit uint64
62+ }
63 }
64-
65- err = json.Unmarshal(body, &stats)
66- if err != nil {
67+ if err = json.Unmarshal(body, &report); err != nil {
68 return stats, err
69 }
70-
71+ if len(report.Stats) != 1 {
72+ return stats, fmt.Errorf("expected one container stats record, got %d", len(report.Stats))
73+ }
74+ value := report.Stats[0]
75+ if value.SystemNano == 0 {
76+ return stats, ContainerWrongState
77+ }
78+ if value.CPUSystemNano > value.CPUNano {
79+ return stats, fmt.Errorf("container system CPU time exceeds total CPU time")
80+ }
81+ stats.CPUStats.CPUUsage.TotalUsage = value.CPUNano
82+ stats.CPUStats.CPUUsage.UsageInKernelmode = value.CPUSystemNano
83+ stats.CPUStats.CPUUsage.UsageInUsermode = value.CPUNano - value.CPUSystemNano
84+ stats.MemoryStats.Usage = value.MemUsage
85+ stats.MemoryStats.Limit = value.MemLimit
86 return stats, nil
87 }
88--- a/handle.go
89+++ b/handle.go
90@@ -104,7 +104,9 @@
91 func (h *TaskHandle) runStatsEmitter(ctx context.Context, statsChannel chan *drivers.TaskResourceUsage, interval time.Duration) {
92 timer := time.NewTimer(0)
93 h.logger.Debug("Starting statsEmitter", "container", h.containerID)
94+ h.stateLock.Lock()
95 h.collectionInterval = interval
96+ h.stateLock.Unlock()
97 for {
98 select {
99 case <-ctx.Done():
100@@ -201,10 +203,12 @@
101 return
102
103 case <-timer.C:
104- timer.Reset(h.collectionInterval)
105 }
106
107 containerStats, statsErr := h.podmanClient.ContainerStats(h.driver.ctx, h.containerID)
108+ h.stateLock.RLock()
109+ timer.Reset(h.collectionInterval)
110+ h.stateLock.RUnlock()
111 if statsErr != nil {
112 gone := false
113 if errors.Is(statsErr, api.ContainerNotFound) {
114--- /dev/null
115+++ b/api/native_stats_test.go
116@@ -0,0 +1,58 @@
117+package api
118+
119+import (
120+ "context"
121+ "errors"
122+ "net/http"
123+ "net/http/httptest"
124+ "testing"
125+
126+ "github.com/hashicorp/go-hclog"
127+)
128+
129+func TestNativeContainerStats(t *testing.T) {
130+ for _, test := range []struct {
131+ name string
132+ status int
133+ body string
134+ gone error
135+ fails bool
136+ }{
137+ {"running", 200, `{"Error":null,"Stats":[{"CPUNano":100,"CPUSystemNano":25,"SystemNano":1000,"MemUsage":64,"MemLimit":256}]}`, nil, false},
138+ {"idle", 200, `{"Error":null,"Stats":[{"CPUNano":0,"CPUSystemNano":0,"SystemNano":1000}]}`, nil, false},
139+ {"exited", 200, `{"Error":null,"Stats":[{"CPUNano":0,"CPUSystemNano":0,"SystemNano":0,"MemUsage":0,"MemLimit":0}]}`, ContainerWrongState, true},
140+ {"missing", 404, `{"cause":"no such container"}`, ContainerNotFound, true},
141+ {"stopped", 404, `{"cause":"container is stopped"}`, ContainerWrongState, true},
142+ {"invalid_state", 404, `{"cause":"container state improper"}`, ContainerWrongState, true},
143+ {"storage_failure", 404, `{"cause":"input/output error"}`, nil, true},
144+ {"upstream_failure", 500, `{"cause":"input/output error"}`, nil, true},
145+ {"truncated", 200, `{"Stats":[`, nil, true},
146+ {"empty", 200, `{"Stats":[]}`, nil, true},
147+ {"invalid_cpu", 200, `{"Stats":[{"CPUNano":10,"CPUSystemNano":20,"SystemNano":1000}]}`, nil, true},
148+ } {
149+ t.Run(test.name, func(t *testing.T) {
150+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
151+ if r.URL.Path != "/v4.0.0/libpod/containers/stats" || r.URL.Query().Get("containers") != "x/y ?" || r.URL.Query().Get("stream") != "false" {
152+ t.Errorf("unexpected stats request: %s", r.URL)
153+ }
154+ w.WriteHeader(test.status)
155+ _, _ = w.Write([]byte(test.body))
156+ }))
157+ defer server.Close()
158+ client := NewClient(hclog.NewNullLogger(), ClientConfig{SocketPath: server.URL})
159+ stats, err := client.ContainerStats(context.Background(), "x/y ?")
160+ if (err != nil) != test.fails {
161+ t.Fatalf("stats error: %v", err)
162+ }
163+ if test.gone != nil && !errors.Is(err, test.gone) {
164+ t.Fatalf("expected %v, got %v", test.gone, err)
165+ }
166+ if test.gone == nil && (errors.Is(err, ContainerNotFound) || errors.Is(err, ContainerWrongState)) {
167+ t.Fatalf("transient failure treated as container exit: %v", err)
168+ }
169+ if test.name == "running" && (stats.CPUStats.CPUUsage.TotalUsage != 100 || stats.CPUStats.CPUUsage.UsageInKernelmode != 25 || stats.CPUStats.CPUUsage.UsageInUsermode != 75 || stats.MemoryStats.Usage != 64 || stats.MemoryStats.Limit != 256) {
170+ t.Fatalf("incorrect resource counters: %+v", stats)
171+ }
172+ })
173+ }
174+}
175--- /dev/null
176+++ b/native_monitor_test.go
177@@ -0,0 +1,57 @@
178+package main
179+
180+import (
181+ "context"
182+ "net/http"
183+ "net/http/httptest"
184+ "testing"
185+ "time"
186+
187+ "github.com/hashicorp/go-hclog"
188+ "github.com/hashicorp/nomad-driver-podman/api"
189+)
190+
191+func TestMonitorWaitsAfterSlowStats(t *testing.T) {
192+ started := make(chan time.Time, 4)
193+ finished := make(chan time.Time, 4)
194+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
195+ started <- time.Now()
196+ time.Sleep(100 * time.Millisecond)
197+ _, _ = w.Write([]byte(`{"Stats":[{"CPUNano":100,"CPUSystemNano":25,"SystemNano":1000}]}`))
198+ finished <- time.Now()
199+ }))
200+ defer server.Close()
201+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
202+ defer cancel()
203+ logger := hclog.NewNullLogger()
204+ handle := TaskHandle{driver: &Driver{ctx: ctx}, logger: logger,
205+ podmanClient: api.NewClient(logger, api.ClientConfig{SocketPath: server.URL}),
206+ collectionInterval: 40 * time.Millisecond}
207+ done := make(chan struct{})
208+ go func() { handle.runContainerMonitor(); close(done) }()
209+ select {
210+ case <-started:
211+ case <-ctx.Done():
212+ t.Fatal("monitor did not start")
213+ }
214+ var firstEnd time.Time
215+ select {
216+ case firstEnd = <-finished:
217+ case <-ctx.Done():
218+ t.Fatal("stats did not finish")
219+ }
220+ select {
221+ case secondStart := <-started:
222+ if secondStart.Sub(firstEnd) < 30*time.Millisecond {
223+ t.Fatal("slow stats caused immediate polling")
224+ }
225+ case <-ctx.Done():
226+ t.Fatal("monitor did not poll again")
227+ }
228+ cancel()
229+ select {
230+ case <-done:
231+ case <-time.After(time.Second):
232+ t.Fatal("monitor did not stop")
233+ }
234+}