| author | wenzelm | 
| Wed, 08 Nov 2023 12:41:41 +0100 | |
| changeset 78914 | 715f1bd21993 | 
| parent 78600 | afb83ba8d24c | 
| child 82918 | af85ea5d9b20 | 
| permissions | -rw-r--r-- | 
| 68758 | 1  | 
/* Title: Pure/PIDE/document_status.scala  | 
2  | 
Author: Makarius  | 
|
3  | 
||
4  | 
Document status based on markup information.  | 
|
5  | 
*/  | 
|
6  | 
||
7  | 
package isabelle  | 
|
8  | 
||
9  | 
||
| 75393 | 10  | 
object Document_Status {
 | 
| 68758 | 11  | 
/* command status */  | 
12  | 
||
| 75393 | 13  | 
  object Command_Status {
 | 
| 68758 | 14  | 
val proper_elements: Markup.Elements =  | 
15  | 
Markup.Elements(Markup.ACCEPTED, Markup.FORKED, Markup.JOINED, Markup.RUNNING,  | 
|
| 
68871
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
16  | 
Markup.FINISHED, Markup.FAILED, Markup.CANCELED)  | 
| 68758 | 17  | 
|
18  | 
val liberal_elements: Markup.Elements =  | 
|
19  | 
proper_elements + Markup.WARNING + Markup.LEGACY + Markup.ERROR  | 
|
20  | 
||
| 75393 | 21  | 
    def make(markup_iterator: Iterator[Markup]): Command_Status = {
 | 
| 68758 | 22  | 
var touched = false  | 
23  | 
var accepted = false  | 
|
24  | 
var warned = false  | 
|
25  | 
var failed = false  | 
|
| 
68871
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
26  | 
var canceled = false  | 
| 
68884
 
9b97d0b20d95
clarified quasi_consolidated state: ensure that exports are present for ok nodes;
 
wenzelm 
parents: 
68883 
diff
changeset
 | 
27  | 
var finalized = false  | 
| 68758 | 28  | 
var forks = 0  | 
29  | 
var runs = 0  | 
|
30  | 
      for (markup <- markup_iterator) {
 | 
|
31  | 
        markup.name match {
 | 
|
32  | 
case Markup.ACCEPTED => accepted = true  | 
|
33  | 
case Markup.FORKED => touched = true; forks += 1  | 
|
34  | 
case Markup.JOINED => forks -= 1  | 
|
35  | 
case Markup.RUNNING => touched = true; runs += 1  | 
|
36  | 
case Markup.FINISHED => runs -= 1  | 
|
37  | 
case Markup.WARNING | Markup.LEGACY => warned = true  | 
|
38  | 
case Markup.FAILED | Markup.ERROR => failed = true  | 
|
| 
68871
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
39  | 
case Markup.CANCELED => canceled = true  | 
| 
68884
 
9b97d0b20d95
clarified quasi_consolidated state: ensure that exports are present for ok nodes;
 
wenzelm 
parents: 
68883 
diff
changeset
 | 
40  | 
case Markup.FINALIZED => finalized = true  | 
| 68758 | 41  | 
case _ =>  | 
42  | 
}  | 
|
43  | 
}  | 
|
| 68887 | 44  | 
Command_Status(  | 
45  | 
touched = touched,  | 
|
46  | 
accepted = accepted,  | 
|
47  | 
warned = warned,  | 
|
48  | 
failed = failed,  | 
|
49  | 
canceled = canceled,  | 
|
50  | 
finalized = finalized,  | 
|
51  | 
forks = forks,  | 
|
52  | 
runs = runs)  | 
|
| 68758 | 53  | 
}  | 
54  | 
||
| 71601 | 55  | 
val empty: Command_Status = make(Iterator.empty)  | 
| 68758 | 56  | 
|
57  | 
def merge(status_iterator: Iterator[Command_Status]): Command_Status =  | 
|
58  | 
      if (status_iterator.hasNext) {
 | 
|
| 73344 | 59  | 
val status0 = status_iterator.next()  | 
| 73359 | 60  | 
status_iterator.foldLeft(status0)(_ + _)  | 
| 68758 | 61  | 
}  | 
62  | 
else empty  | 
|
63  | 
}  | 
|
64  | 
||
65  | 
sealed case class Command_Status(  | 
|
66  | 
private val touched: Boolean,  | 
|
67  | 
private val accepted: Boolean,  | 
|
68  | 
private val warned: Boolean,  | 
|
69  | 
private val failed: Boolean,  | 
|
| 
68871
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
70  | 
private val canceled: Boolean,  | 
| 
68884
 
9b97d0b20d95
clarified quasi_consolidated state: ensure that exports are present for ok nodes;
 
wenzelm 
parents: 
68883 
diff
changeset
 | 
71  | 
private val finalized: Boolean,  | 
| 68758 | 72  | 
forks: Int,  | 
| 75393 | 73  | 
runs: Int  | 
74  | 
  ) {
 | 
|
| 68758 | 75  | 
def + (that: Command_Status): Command_Status =  | 
76  | 
Command_Status(  | 
|
| 68887 | 77  | 
touched = touched || that.touched,  | 
78  | 
accepted = accepted || that.accepted,  | 
|
79  | 
warned = warned || that.warned,  | 
|
80  | 
failed = failed || that.failed,  | 
|
81  | 
canceled = canceled || that.canceled,  | 
|
82  | 
finalized = finalized || that.finalized,  | 
|
83  | 
forks = forks + that.forks,  | 
|
84  | 
runs = runs + that.runs)  | 
|
| 68758 | 85  | 
|
86  | 
def is_unprocessed: Boolean = accepted && !failed && (!touched || (forks != 0 && runs == 0))  | 
|
87  | 
def is_running: Boolean = runs != 0  | 
|
88  | 
def is_warned: Boolean = warned  | 
|
89  | 
def is_failed: Boolean = failed  | 
|
90  | 
def is_finished: Boolean = !failed && touched && forks == 0 && runs == 0  | 
|
| 
68871
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
91  | 
def is_canceled: Boolean = canceled  | 
| 
68884
 
9b97d0b20d95
clarified quasi_consolidated state: ensure that exports are present for ok nodes;
 
wenzelm 
parents: 
68883 
diff
changeset
 | 
92  | 
def is_finalized: Boolean = finalized  | 
| 68881 | 93  | 
def is_terminated: Boolean = canceled || touched && forks == 0 && runs == 0  | 
| 68758 | 94  | 
}  | 
95  | 
||
96  | 
||
97  | 
/* node status */  | 
|
98  | 
||
| 75393 | 99  | 
  object Node_Status {
 | 
| 
76716
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
100  | 
val empty: Node_Status =  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
101  | 
Node_Status(  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
102  | 
is_suppressed = false,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
103  | 
unprocessed = 0,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
104  | 
running = 0,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
105  | 
warned = 0,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
106  | 
failed = 0,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
107  | 
finished = 0,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
108  | 
canceled = false,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
109  | 
terminated = false,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
110  | 
initialized = false,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
111  | 
finalized = false,  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
112  | 
consolidated = false)  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
113  | 
|
| 68758 | 114  | 
def make(  | 
115  | 
state: Document.State,  | 
|
116  | 
version: Document.Version,  | 
|
| 76714 | 117  | 
name: Document.Node.Name  | 
118  | 
    ): Node_Status = {
 | 
|
| 68758 | 119  | 
val node = version.nodes(name)  | 
120  | 
||
121  | 
var unprocessed = 0  | 
|
122  | 
var running = 0  | 
|
123  | 
var warned = 0  | 
|
124  | 
var failed = 0  | 
|
125  | 
var finished = 0  | 
|
| 
68871
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
126  | 
var canceled = false  | 
| 68892 | 127  | 
var terminated = true  | 
| 
68884
 
9b97d0b20d95
clarified quasi_consolidated state: ensure that exports are present for ok nodes;
 
wenzelm 
parents: 
68883 
diff
changeset
 | 
128  | 
var finalized = false  | 
| 68758 | 129  | 
      for (command <- node.commands.iterator) {
 | 
130  | 
val states = state.command_states(version, command)  | 
|
131  | 
val status = Command_Status.merge(states.iterator.map(_.document_status))  | 
|
132  | 
||
133  | 
if (status.is_running) running += 1  | 
|
134  | 
else if (status.is_failed) failed += 1  | 
|
135  | 
else if (status.is_warned) warned += 1  | 
|
136  | 
else if (status.is_finished) finished += 1  | 
|
137  | 
else unprocessed += 1  | 
|
| 
68871
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
138  | 
|
| 
 
f5c76072db55
more explicit status for "canceled" command within theory node;
 
wenzelm 
parents: 
68770 
diff
changeset
 | 
139  | 
if (status.is_canceled) canceled = true  | 
| 68892 | 140  | 
if (!status.is_terminated) terminated = false  | 
| 
68884
 
9b97d0b20d95
clarified quasi_consolidated state: ensure that exports are present for ok nodes;
 
wenzelm 
parents: 
68883 
diff
changeset
 | 
141  | 
if (status.is_finalized) finalized = true  | 
| 68758 | 142  | 
}  | 
143  | 
val initialized = state.node_initialized(version, name)  | 
|
144  | 
val consolidated = state.node_consolidated(version, name)  | 
|
145  | 
||
| 68887 | 146  | 
Node_Status(  | 
| 
69818
 
60d0ee8f2ddb
more robust: avoid potentially unrelated snapshot for the sake of is_suppressed;
 
wenzelm 
parents: 
69817 
diff
changeset
 | 
147  | 
is_suppressed = version.nodes.is_suppressed(name),  | 
| 68887 | 148  | 
unprocessed = unprocessed,  | 
149  | 
running = running,  | 
|
150  | 
warned = warned,  | 
|
151  | 
failed = failed,  | 
|
152  | 
finished = finished,  | 
|
153  | 
canceled = canceled,  | 
|
154  | 
terminated = terminated,  | 
|
| 68889 | 155  | 
initialized = initialized,  | 
| 68887 | 156  | 
finalized = finalized,  | 
157  | 
consolidated = consolidated)  | 
|
| 68758 | 158  | 
}  | 
159  | 
}  | 
|
160  | 
||
161  | 
sealed case class Node_Status(  | 
|
| 
69818
 
60d0ee8f2ddb
more robust: avoid potentially unrelated snapshot for the sake of is_suppressed;
 
wenzelm 
parents: 
69817 
diff
changeset
 | 
162  | 
is_suppressed: Boolean,  | 
| 68887 | 163  | 
unprocessed: Int,  | 
164  | 
running: Int,  | 
|
165  | 
warned: Int,  | 
|
166  | 
failed: Int,  | 
|
167  | 
finished: Int,  | 
|
168  | 
canceled: Boolean,  | 
|
169  | 
terminated: Boolean,  | 
|
| 68889 | 170  | 
initialized: Boolean,  | 
| 68887 | 171  | 
finalized: Boolean,  | 
| 75393 | 172  | 
consolidated: Boolean  | 
173  | 
  ) {
 | 
|
| 
76716
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
174  | 
def is_empty: Boolean = this == Node_Status.empty  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
175  | 
|
| 68758 | 176  | 
def ok: Boolean = failed == 0  | 
177  | 
def total: Int = unprocessed + running + warned + failed + finished  | 
|
178  | 
||
| 
69844
 
b21ddfa7042b
clarified quasi_consolidated status after 5f160df596c1 -- relevant for headless PIDE session (e.g. "isabelle dump");
 
wenzelm 
parents: 
69818 
diff
changeset
 | 
179  | 
def quasi_consolidated: Boolean = !is_suppressed && !finalized && terminated  | 
| 68897 | 180  | 
|
| 68898 | 181  | 
def percentage: Int =  | 
182  | 
if (consolidated) 100  | 
|
183  | 
else if (total == 0) 0  | 
|
184  | 
else (((total - unprocessed).toDouble / total) * 100).toInt min 99  | 
|
185  | 
||
| 68758 | 186  | 
def json: JSON.Object.T =  | 
187  | 
      JSON.Object("ok" -> ok, "total" -> total, "unprocessed" -> unprocessed,
 | 
|
188  | 
"running" -> running, "warned" -> warned, "failed" -> failed, "finished" -> finished,  | 
|
| 68898 | 189  | 
"canceled" -> canceled, "consolidated" -> consolidated,  | 
190  | 
"percentage" -> percentage)  | 
|
| 68758 | 191  | 
}  | 
192  | 
||
193  | 
||
| 69863 | 194  | 
/* overall timing */  | 
| 68758 | 195  | 
|
| 75393 | 196  | 
  object Overall_Timing {
 | 
| 69863 | 197  | 
val empty: Overall_Timing = Overall_Timing(0.0, Map.empty)  | 
| 68758 | 198  | 
|
199  | 
def make(  | 
|
200  | 
state: Document.State,  | 
|
201  | 
version: Document.Version,  | 
|
| 69863 | 202  | 
commands: Iterable[Command],  | 
| 75393 | 203  | 
threshold: Double = 0.0  | 
204  | 
    ): Overall_Timing = {
 | 
|
| 68758 | 205  | 
var total = 0.0  | 
| 69863 | 206  | 
var command_timings = Map.empty[Command, Double]  | 
| 68758 | 207  | 
      for {
 | 
| 69863 | 208  | 
command <- commands.iterator  | 
| 68758 | 209  | 
st <- state.command_states(version, command)  | 
210  | 
      } {
 | 
|
211  | 
val command_timing =  | 
|
| 73359 | 212  | 
          st.status.foldLeft(0.0) {
 | 
| 68758 | 213  | 
case (timing, Markup.Timing(t)) => timing + t.elapsed.seconds  | 
214  | 
case (timing, _) => timing  | 
|
| 73359 | 215  | 
}  | 
| 68758 | 216  | 
total += command_timing  | 
| 69864 | 217  | 
        if (command_timing > 0.0 && command_timing >= threshold) {
 | 
218  | 
command_timings += (command -> command_timing)  | 
|
219  | 
}  | 
|
| 68758 | 220  | 
}  | 
| 69863 | 221  | 
Overall_Timing(total, command_timings)  | 
| 68758 | 222  | 
}  | 
223  | 
}  | 
|
224  | 
||
| 75393 | 225  | 
  sealed case class Overall_Timing(total: Double, command_timings: Map[Command, Double]) {
 | 
| 69864 | 226  | 
def command_timing(command: Command): Double =  | 
227  | 
command_timings.getOrElse(command, 0.0)  | 
|
228  | 
}  | 
|
| 68759 | 229  | 
|
230  | 
||
231  | 
/* nodes status */  | 
|
232  | 
||
| 78600 | 233  | 
  enum Overall_Node_Status { case ok, failed, pending }
 | 
| 68759 | 234  | 
|
| 75393 | 235  | 
  object Nodes_Status {
 | 
| 68903 | 236  | 
val empty: Nodes_Status = new Nodes_Status(Map.empty, Document.Nodes.empty)  | 
| 68759 | 237  | 
}  | 
238  | 
||
| 68903 | 239  | 
final class Nodes_Status private(  | 
240  | 
private val rep: Map[Document.Node.Name, Node_Status],  | 
|
| 75393 | 241  | 
nodes: Document.Nodes  | 
242  | 
  ) {
 | 
|
| 
68770
 
add44e2b8cb0
optional notification of nodes_status (via progress);
 
wenzelm 
parents: 
68769 
diff
changeset
 | 
243  | 
def is_empty: Boolean = rep.isEmpty  | 
| 
 
add44e2b8cb0
optional notification of nodes_status (via progress);
 
wenzelm 
parents: 
68769 
diff
changeset
 | 
244  | 
def apply(name: Document.Node.Name): Node_Status = rep(name)  | 
| 68759 | 245  | 
def get(name: Document.Node.Name): Option[Node_Status] = rep.get(name)  | 
246  | 
||
| 
76716
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
247  | 
def present(  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
248  | 
domain: Option[List[Document.Node.Name]] = None  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
249  | 
    ): List[(Document.Node.Name, Node_Status)] = {
 | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
250  | 
for (name <- domain.getOrElse(nodes.topological_order))  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
251  | 
yield name -> get(name).getOrElse(Node_Status.empty)  | 
| 
 
a7602257a825
clarified state document nodes for Theories_Status / Document_Dockable;
 
wenzelm 
parents: 
76714 
diff
changeset
 | 
252  | 
}  | 
| 68903 | 253  | 
|
| 
68884
 
9b97d0b20d95
clarified quasi_consolidated state: ensure that exports are present for ok nodes;
 
wenzelm 
parents: 
68883 
diff
changeset
 | 
254  | 
def quasi_consolidated(name: Document.Node.Name): Boolean =  | 
| 
68883
 
3653b3ad729e
clarified Thy_Resources.Session.use_theories: "terminated" node status is sufficient;
 
wenzelm 
parents: 
68881 
diff
changeset
 | 
255  | 
      rep.get(name) match {
 | 
| 68897 | 256  | 
case Some(st) => st.quasi_consolidated  | 
| 
68883
 
3653b3ad729e
clarified Thy_Resources.Session.use_theories: "terminated" node status is sufficient;
 
wenzelm 
parents: 
68881 
diff
changeset
 | 
257  | 
case None => false  | 
| 
 
3653b3ad729e
clarified Thy_Resources.Session.use_theories: "terminated" node status is sufficient;
 
wenzelm 
parents: 
68881 
diff
changeset
 | 
258  | 
}  | 
| 
 
3653b3ad729e
clarified Thy_Resources.Session.use_theories: "terminated" node status is sufficient;
 
wenzelm 
parents: 
68881 
diff
changeset
 | 
259  | 
|
| 78600 | 260  | 
def overall_node_status(name: Document.Node.Name): Overall_Node_Status =  | 
| 68759 | 261  | 
      rep.get(name) match {
 | 
262  | 
case Some(st) if st.consolidated =>  | 
|
263  | 
if (st.ok) Overall_Node_Status.ok else Overall_Node_Status.failed  | 
|
264  | 
case _ => Overall_Node_Status.pending  | 
|
265  | 
}  | 
|
266  | 
||
| 68763 | 267  | 
def update(  | 
| 
69255
 
800b1ce96fce
more general support for Isabelle/PIDE file formats -- less hardwired Bibtex operations;
 
wenzelm 
parents: 
68904 
diff
changeset
 | 
268  | 
resources: Resources,  | 
| 68763 | 269  | 
state: Document.State,  | 
270  | 
version: Document.Version,  | 
|
| 68769 | 271  | 
domain: Option[Set[Document.Node.Name]] = None,  | 
| 75393 | 272  | 
trim: Boolean = false  | 
273  | 
    ): (Boolean, Nodes_Status) = {
 | 
|
| 68903 | 274  | 
val nodes1 = version.nodes  | 
| 68763 | 275  | 
val update_iterator =  | 
276  | 
        for {
 | 
|
| 68903 | 277  | 
name <- domain.getOrElse(nodes1.domain).iterator  | 
| 74756 | 278  | 
if !Resources.hidden_node(name) && !resources.session_base.loaded_theory(name)  | 
| 68763 | 279  | 
st = Document_Status.Node_Status.make(state, version, name)  | 
280  | 
if !rep.isDefinedAt(name) || rep(name) != st  | 
|
281  | 
} yield (name -> st)  | 
|
282  | 
val rep1 = rep ++ update_iterator  | 
|
| 68903 | 283  | 
val rep2 = if (trim) rep1 -- rep1.keysIterator.filterNot(nodes1.domain) else rep1  | 
| 68764 | 284  | 
|
| 68903 | 285  | 
(rep != rep2, new Nodes_Status(rep2, nodes1))  | 
| 68763 | 286  | 
}  | 
| 68761 | 287  | 
|
| 68759 | 288  | 
override def hashCode: Int = rep.hashCode  | 
289  | 
override def equals(that: Any): Boolean =  | 
|
290  | 
      that match {
 | 
|
291  | 
case other: Nodes_Status => rep == other.rep  | 
|
292  | 
case _ => false  | 
|
293  | 
}  | 
|
| 68765 | 294  | 
|
| 75393 | 295  | 
    override def toString: String = {
 | 
| 68765 | 296  | 
var ok = 0  | 
297  | 
var failed = 0  | 
|
298  | 
var pending = 0  | 
|
| 68873 | 299  | 
var canceled = 0  | 
| 68765 | 300  | 
      for (name <- rep.keysIterator) {
 | 
301  | 
        overall_node_status(name) match {
 | 
|
302  | 
case Overall_Node_Status.ok => ok += 1  | 
|
303  | 
case Overall_Node_Status.failed => failed += 1  | 
|
304  | 
case Overall_Node_Status.pending => pending += 1  | 
|
305  | 
}  | 
|
| 68873 | 306  | 
if (apply(name).canceled) canceled += 1  | 
| 68765 | 307  | 
}  | 
| 68873 | 308  | 
"Nodes_Status(ok = " + ok + ", failed = " + failed + ", pending = " + pending +  | 
309  | 
", canceled = " + canceled + ")"  | 
|
| 68765 | 310  | 
}  | 
| 68759 | 311  | 
}  | 
| 68758 | 312  | 
}  |