1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
package main
import (
"fmt"
"strings"
"time"
"go.wit.com/lib/gui/shell"
"go.wit.com/log"
)
func (h *HyperT) pollHypervisor() {
url := "http://" + h.pb.Hostname + ":2520/vms"
log.Log(POLL, "wget url =", url)
s := shell.Wget(url)
if s == nil {
return
}
var bytesSplice []byte
bytesSplice = s.Bytes()
// fmt.Fprintln(w, string(bytesSplice))
for _, line := range strings.Split(string(bytesSplice), "\n") {
if line == "" {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
state := fields[0]
name := fields[1]
if state == "ON" {
log.Log(POLL, h.pb.Hostname, "STATE:", state, "HOST:", name, "rest:", fields[2:])
d := findDroplet(name)
if d == nil {
// this is a new unknown droplet (not in the config file)
d = new(DropletT)
d.pb.Hostname = name
d.h = h
d.lastpoll = time.Now()
d.CurrentState = "ON"
me.droplets = append(me.droplets, d)
log.Log(EVENT, name, "IS NEW. ADDED ON", h.pb.Hostname)
}
log.Log(INFO, "ALREADY RECORDED", d.pb.Hostname)
// update the status to ON and the last polled value
d.CurrentState = "ON"
d.lastpoll = time.Now()
if d.h == nil {
// this means the droplet was in the config file
// but this is the first time it's shown up as running
// this should mean a droplet is running where the config file says it probably should be running
if d.pb.PreferredHypervisor == h.pb.Hostname {
log.Log(EVENT, "new droplet", d.pb.Hostname, "(matches config hypervisor", h.pb.Hostname+")")
d.h = h
continue
}
log.Log(EVENT, "new droplet", d.pb.Hostname, "on", h.pb.Hostname, "(in config file without preferred hypervisor)")
d.h = h
continue
}
// this means the droplet is still where it was before
if d.h.pb.Hostname != h.pb.Hostname {
log.Log(EVENT, "droplet", d.h.pb.Hostname, "moved to", h.pb.Hostname)
continue
}
d.h = h
}
continue
}
h.lastpoll = time.Now()
h.killcount = 0 // poll worked. reset killcount
}
func findDroplet(name string) *DropletT {
for _, d := range me.droplets {
if d.pb.Hostname == name {
return d
}
}
return nil
}
func findHypervisor(name string) *HyperT {
for _, h := range me.hypers {
if h.pb.Hostname == name {
return h
}
}
return nil
}
// check the state of the cluster and return a string
// that is intended to be sent to an uptime monitor like Kuma
func clusterHealthy() (bool, string) {
var good bool = true
var total int
var working int
var failed int
var missing int
var unknown int
var unknownList []string
for _, d := range me.droplets {
total += 1
if d.pb.StartState != "ON" {
continue
}
dur := time.Since(d.lastpoll) // Calculate the elapsed time
if d.CurrentState == "" {
// log.Info("SKIP. hostname has not been polled yet", d.pb.Hostname, d.hname)
unknown += 1
unknownList = append(unknownList, d.pb.Hostname)
continue
}
var hname string
if d.h != nil {
hname = d.h.pb.Hostname
}
if d.CurrentState != "ON" {
log.Info("BAD STATE", d.pb.StartState, d.pb.Hostname, hname, "CurrentState =", d.CurrentState, shell.FormatDuration(dur))
good = false
failed += 1
} else {
dur := time.Since(d.lastpoll) // Calculate the elapsed time
if dur > time.Minute {
log.Info("GOOD STATE MISSING", d.pb.Hostname, hname, shell.FormatDuration(dur))
good = false
d.CurrentState = "MISSING"
failed += 1
continue
}
l := shell.FormatDuration(dur)
if l == "" {
log.Info("DUR IS EMPTY", dur)
missing += 1
continue
}
working += 1
// log.Info("GOOD STATE ON", d.pb.Hostname, d.hname, "dur =", l)
}
}
var summary string = "("
summary += fmt.Sprintf("total = %d ", total)
summary += fmt.Sprintf("working = %d ", working)
if missing > 0 {
summary += fmt.Sprintf("missing = %d ", missing)
}
if unknown > 0 {
summary += fmt.Sprintf("unknown = %d ", unknown, unknownList)
}
if failed > 0 {
summary += fmt.Sprintf("failed = %d ", failed)
}
summary = strings.TrimSpace(summary)
summary += ")"
if me.killcount > 0 {
summary += "(killcount=" + fmt.Sprintf("%d", me.killcount) + ")"
}
last := time.Since(me.unstable)
if last > 133*time.Second {
// the cluster has not been stable for 10 seconds
s := strings.TrimSpace(shell.FormatDuration(last))
summary += "(stable=" + s + ")"
}
if good {
return good, "GOOD=true " + summary
}
me.unstable = time.Now()
return good, "GOOD=false " + summary
}
|
