Skip to content

Commit 76023df

Browse files
ryannewingtonryannewington
committed
Adds UI notification when the controller crashes
Fixes an issue with the controller mechanism crashing when dequeuing items
1 parent 651e47d commit 76023df

File tree

7 files changed

+111
-124
lines changed

7 files changed

+111
-124
lines changed

‎src/Lithnet.Miiserver.AutoSync/Enums/ErrorState.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
using System.Runtime.Serialization;
2+
3+
namespace Lithnet.Miiserver.AutoSync
4+
{
5+
[DataContract]
6+
public enum ErrorState
7+
{
8+
[EnumMember]
9+
None = 0,
10+
11+
[EnumMember]
12+
ThresholdExceeded = 1,
13+
14+
[EnumMember]
15+
ControllerFaulted = 2,
16+
17+
[EnumMember]
18+
UnexpectedChange = 3,
19+
}
20+
}

‎src/Lithnet.Miiserver.AutoSync/ExecutionEngine.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ public void Stop(Guid managementAgentID, bool cancelRun)
156156

157157
lock (e)
158158
{
159-
e.Stop(cancelRun, true, false);
159+
e.Stop(cancelRun, true, ErrorState.None);
160160
}
161161
}
162162

@@ -296,7 +296,7 @@ private void StopMAControllers(bool cancelRun)
296296
lock (e)
297297
{
298298
Thread.CurrentThread.SetThreadName($"Stop controller {e.ManagementAgentName}");
299-
e.Stop(cancelRun, true, false);
299+
e.Stop(cancelRun, true, ErrorState.None);
300300
}
301301
}
302302
catch (OperationCanceledException)

‎src/Lithnet.Miiserver.AutoSync/Lithnet.Miiserver.AutoSync.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
<Compile Include="ConfigService\IConfigService.cs" />
9999
<Compile Include="Config\ConfigFile.cs" />
100100
<Compile Include="Config\Settings.cs" />
101+
<Compile Include="Enums\ErrorState.cs" />
101102
<Compile Include="Enums\ExecutionErrorBehaviour.cs" />
102103
<Compile Include="Enums\AutoImportScheduling.cs" />
103104
<Compile Include="Enums\ControlState.cs" />

‎src/Lithnet.Miiserver.AutoSync/MAInterface/ActionQueue.cs‎

Lines changed: 42 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,15 @@
11
using System;
2-
using System.Collections.Concurrent;
32
using System.Collections.Generic;
43
using System.Linq;
5-
using System.Text;
64
using System.Threading;
7-
using System.Threading.Tasks;
85

96
namespace Lithnet.Miiserver.AutoSync
107
{
11-
internal class ActionQueue<T>
8+
internal class ActionQueue<T> where T : ExecutionParameters
129
{
1310
private List<T> queue;
1411
private readonly SemaphoreSlim itemNotification;
15-
private readonly SemaphoreSlim completeAdding;
12+
private readonly CancellationTokenSource completeAddingCancellationToken;
1613
private bool complete;
1714
private object lockItem;
1815

@@ -22,67 +19,10 @@ public ActionQueue()
2219
this.lockItem = new object();
2320
this.queue = new List<T>();
2421
this.itemNotification = new SemaphoreSlim(0);
25-
this.completeAdding = new SemaphoreSlim(0);
22+
this.completeAddingCancellationToken = new CancellationTokenSource();
2623
}
2724

28-
public void Add(T item)
29-
{
30-
if (this.complete)
31-
{
32-
throw new InvalidOperationException();
33-
}
34-
35-
lock (this.lockItem)
36-
{
37-
this.queue.Add(item);
38-
}
39-
40-
this.itemNotification.Release();
41-
}
42-
43-
public void AddToFront(T item)
44-
{
45-
if (this.complete)
46-
{
47-
throw new InvalidOperationException();
48-
}
49-
50-
lock (this.lockItem)
51-
{
52-
this.queue.Insert(0, item);
53-
}
54-
55-
this.itemNotification.Release();
56-
}
57-
58-
public void MoveToFront(T item)
59-
{
60-
if (this.complete)
61-
{
62-
throw new InvalidOperationException();
63-
}
64-
65-
lock (this.lockItem)
66-
{
67-
this.queue.Remove(item);
68-
this.queue.Insert(0, item);
69-
}
70-
}
71-
72-
public bool Contains(T item)
73-
{
74-
if (this.complete)
75-
{
76-
throw new InvalidOperationException();
77-
}
78-
79-
lock (this.lockItem)
80-
{
81-
return this.queue.Contains(item);
82-
}
83-
}
84-
85-
public bool MoveToFrontIfExists(T item)
25+
public bool Add(T item)
8626
{
8727
if (this.complete)
8828
{
@@ -93,18 +33,34 @@ public bool MoveToFrontIfExists(T item)
9333
{
9434
if (this.queue.Contains(item))
9535
{
96-
if (this.queue.Count == 1)
36+
if (item.RunImmediate)
9737
{
98-
return false;
38+
if (this.queue.Count == 1)
39+
{
40+
return false;
41+
}
42+
43+
this.queue.Remove(item);
44+
this.queue.Insert(0, item);
9945
}
10046

101-
this.queue.Remove(item);
102-
this.queue.Insert(0, item);
103-
return true;
47+
return false;
48+
}
49+
else
50+
{
51+
if (item.RunImmediate)
52+
{
53+
this.queue.Insert(0, item);
54+
}
55+
else
56+
{
57+
this.queue.Add(item);
58+
}
10459
}
10560
}
10661

107-
return false;
62+
this.itemNotification.Release();
63+
return true;
10864
}
10965

11066
public void CompleteAdding()
@@ -115,34 +71,44 @@ public void CompleteAdding()
11571
}
11672

11773
this.complete = true;
118-
this.completeAdding.Release();
74+
this.completeAddingCancellationToken.Cancel();
11975
}
12076

12177
public IEnumerable<T> Consume(CancellationToken token)
12278
{
12379
while (!token.IsCancellationRequested)
12480
{
125-
WaitHandle.WaitAny(new[] { this.itemNotification.AvailableWaitHandle, this.completeAdding.AvailableWaitHandle, token.WaitHandle });
81+
this.itemNotification.Wait(CancellationTokenSource.CreateLinkedTokenSource(token, this.completeAddingCancellationToken.Token).Token);
12682

12783
if (this.complete || token.IsCancellationRequested)
12884
{
12985
yield break;
13086
}
13187

88+
T item = null;
89+
13290
lock (this.lockItem)
13391
{
134-
T item = this.queue[0];
135-
this.queue.RemoveAt(0);
92+
if (this.queue.Count > 0)
93+
{
94+
item = this.queue[0];
95+
this.queue.RemoveAt(0);
96+
}
97+
}
98+
99+
if (item != null)
100+
{
136101
yield return item;
137102
}
138103
}
139104
}
140105

141-
public T[] ToArray()
106+
public override string ToString()
142107
{
143108
lock (this.lockItem)
144109
{
145-
return this.queue.ToArray();
110+
return string.Join(", ", this.queue.Select(t => t.RunProfileName)
111+
);
146112
}
147113
}
148114
}

‎src/Lithnet.Miiserver.AutoSync/MAInterface/MAController.cs‎

Lines changed: 22 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1084,7 +1084,7 @@ private void PerformPostRunActions(RunDetails r)
10841084

10851085
if (this.controllerScript.HasStoppedMA)
10861086
{
1087-
this.Stop(false, false, false);
1087+
this.Stop(false, false, ErrorState.UnexpectedChange);
10881088
return;
10891089
}
10901090

@@ -1164,7 +1164,7 @@ private void ProcessUnexpectedChangeException(UnexpectedChangeException ex)
11641164
else
11651165
{
11661166
this.LogWarn($"Controller indicated that management agent controller should stop further processing on this MA. Run Profile {this.ExecutingRunProfile}");
1167-
this.Stop(false, false, false);
1167+
this.Stop(false, false, ErrorState.UnexpectedChange);
11681168
}
11691169
}
11701170

@@ -1199,7 +1199,7 @@ public void Start(MAControllerConfiguration config)
11991199
this.WaitAndTakeLock(this.serviceControlLock, nameof(this.serviceControlLock), this.controllerCancellationTokenSource);
12001200
gotLock = true;
12011201
this.ControlState = ControlState.Starting;
1202-
this.InternalStatus.ThresholdExceeded = false;
1202+
this.InternalStatus.ErrorState = ErrorState.None;
12031203
this.pendingActions = new ActionQueue<ExecutionParameters>();
12041204
this.perProfileLastRunStatus = new Dictionary<string, string>();
12051205

@@ -1236,6 +1236,7 @@ public void Start(MAControllerConfiguration config)
12361236
catch (Exception ex)
12371237
{
12381238
this.LogError(ex, "The controller encountered a unrecoverable error");
1239+
this.Stop(false, false, ErrorState.ControllerFaulted);
12391240
}
12401241
}, this.controllerCancellationTokenSource.Token);
12411242

@@ -1246,7 +1247,7 @@ public void Start(MAControllerConfiguration config)
12461247
catch (Exception ex)
12471248
{
12481249
logger.Error(ex, "An error occurred starting the controller");
1249-
this.Stop(false, false, false);
1250+
this.Stop(false, false, ErrorState.ControllerFaulted);
12501251
this.Message = $"Startup error: {ex.Message}";
12511252
}
12521253
finally
@@ -1282,7 +1283,7 @@ private void TryCancelRun()
12821283
}
12831284
}
12841285

1285-
public void Stop(bool cancelRun, bool waitForInternalTask, bool thresholdExceeded)
1286+
public void Stop(bool cancelRun, bool waitForInternalTask, ErrorState errorState = ErrorState.None)
12861287
{
12871288
bool gotLock = false;
12881289

@@ -1355,9 +1356,9 @@ public void Stop(bool cancelRun, bool waitForInternalTask, bool thresholdExceede
13551356
this.InternalStatus.Clear();
13561357
this.ControlState = ControlState.Stopped;
13571358

1358-
if (thresholdExceeded)
1359+
if (errorState != ErrorState.None)
13591360
{
1360-
this.InternalStatus.ThresholdExceeded = true;
1361+
this.InternalStatus.ErrorState = errorState;
13611362
this.RaiseStateChange();
13621363
}
13631364

@@ -1418,7 +1419,7 @@ private void Init()
14181419
{
14191420
this.LogWarn($"Threshold was exceeded on management agent run profile {this.ExecutingRunProfile}. The controller will be stopped\n{ex.Message}");
14201421
this.SendThresholdExceededMail(ex.RunDetails, ex.Message);
1421-
this.Stop(false, false, true);
1422+
this.Stop(false, false, ErrorState.ThresholdExceeded);
14221423
throw;
14231424
}
14241425
catch (Exception ex)
@@ -1618,7 +1619,7 @@ private void TakeLocksAndExecuteOld(ExecutionParameters action)
16181619
{
16191620
this.LogWarn($"Threshold was exceeded on management agent run profile {this.ExecutingRunProfile}. The controller will be stopped\n{ex.Message}");
16201621
this.SendThresholdExceededMail(ex.RunDetails, ex.Message);
1621-
this.Stop(false, false, true);
1622+
this.Stop(false, false, ErrorState.ThresholdExceeded);
16221623
}
16231624
finally
16241625
{
@@ -1772,7 +1773,7 @@ private void TakeLocksAndExecuteNew(ExecutionParameters action)
17721773
{
17731774
this.LogWarn($"Threshold was exceeded on management agent run profile {this.ExecutingRunProfile}. The controller will be stopped\n{ex.Message}");
17741775
this.SendThresholdExceededMail(ex.RunDetails, ex.Message);
1775-
this.Stop(false, false, true);
1776+
this.Stop(false, false, ErrorState.ThresholdExceeded);
17761777
}
17771778
finally
17781779
{
@@ -2114,43 +2115,23 @@ internal void AddPendingActionIfNotQueued(ExecutionParameters p, string source)
21142115
}
21152116
}
21162117

2117-
if (this.pendingActions.Contains(p))
2118-
{
2119-
if (p.RunImmediate)
2120-
{
2121-
if (this.pendingActions.MoveToFrontIfExists(p))
2122-
{
2123-
this.LogInfo($"Moved {p.RunProfileName} to the front of the execution queue");
2124-
return;
2125-
}
2126-
}
2127-
2128-
this.LogInfo($"{p.RunProfileName} requested by {source} was ignored because the run profile was already queued");
2129-
return;
2130-
}
2131-
2132-
// Removing this as it may caused changes to go unseen. e.g an import is in progress,
2133-
// a snapshot is taken, but new items become available during the import of the snapshot
2134-
2135-
//if (p.RunProfileName.Equals(this.ExecutingRunProfile, StringComparison.OrdinalIgnoreCase))
2136-
//{
2137-
// this.Trace($"Ignoring queue request for {p.RunProfileName} as it is currently executing");
2138-
// return;
2139-
//}
2140-
21412118
p.QueueID = Interlocked.Increment(ref MAController.CurrentQueueID);
2142-
21432119
this.Trace($"Got queue request for {p.RunProfileName} with id {p.QueueID}");
21442120

2145-
if (p.RunImmediate)
2121+
if (this.pendingActions.Add(p))
21462122
{
2147-
this.pendingActions.AddToFront(p);
2148-
this.LogInfo($"Added {p.RunProfileName} to the front of the execution queue (triggered by: {source})");
2123+
if (p.RunImmediate)
2124+
{
2125+
this.LogInfo($"Added {p.RunProfileName} to the front of the execution queue (triggered by: {source})");
2126+
}
2127+
else
2128+
{
2129+
this.LogInfo($"Added {p.RunProfileName} to the execution queue (triggered by: {source})");
2130+
}
21492131
}
21502132
else
21512133
{
2152-
this.pendingActions.Add(p);
2153-
this.LogInfo($"Added {p.RunProfileName} to the execution queue (triggered by: {source})");
2134+
this.LogInfo($"{p.RunProfileName} requested by {source} was ignored because the run profile was already queued");
21542135
}
21552136

21562137
this.counters.CurrentQueueLength.Increment();
@@ -2173,7 +2154,7 @@ private string GetQueueItemNames(bool includeExecuting = true)
21732154
// the event an add or remove is in progress. Other functions such as ToList are generic and can cause
21742155
// collection modified exceptions when enumerating the values
21752156

2176-
string queuedNames = string.Join(",", this.pendingActions.ToArray().Select(t => t.RunProfileName));
2157+
string queuedNames = this.pendingActions.ToString();
21772158

21782159
if (includeExecuting && this.ExecutingRunProfile != null)
21792160
{

‎src/Lithnet.Miiserver.AutoSync/MAInterface/MAStatus.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public string DisplayState
5959
public bool HasForeignLock { get; internal set; }
6060

6161
[DataMember]
62-
public bool ThresholdExceeded { get; internal set; }
62+
public ErrorState ErrorState { get; internal set; }
6363

6464
public void Clear()
6565
{

0 commit comments

Comments
 (0)