Add RecoveryMode for the RDP Active

Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py

@@ -441,3 +441,17 @@ def Update(inGSettings):
         inGSettings["ServerDict"]["AgentLoopSleepSecFloat"] = 2.0
         if lL: lL.warning(
             f"Backward compatibility (v1.2.1 to v1.2.2): Add key AgentActivityLifetimeSecFloat, AgentConnectionLifetimeSecFloat, AgentLoopSleepSecFloat in ProcessorDict > ServerDict")  # Log about compatibility
+    # Add new key RecoveryDict in ProcessorDict > RobotRDPActive
+    if "RecoveryDict" not in inGSettings["RobotRDPActive"]:
+        inGSettings["RobotRDPActive"]["RecoveryDict"] = {
+            "CatchPeriodSecFloat": 1200, # Catch last 10 minutes
+            "TriggerCountInt": 10, # Activate trigger if for the period orch will catch the reconnect RDP n times
+            "DoDict": {
+                "OSRemotePCRestart": True # Do powershell remote restart
+            },
+            "__StatisticsDict__": {
+                # RDPSessionKeyStr : [time.time(), time.time()],
+            }
+        }
+        if lL: lL.warning(
+            f"Backward compatibility (v1.2.1 to v1.2.2): Add new key RecoveryDict in ProcessorDict > RobotRDPActive")  # Log about compatibility

Sources/pyOpenRPA/Orchestrator/RobotRDPActive/Recovery.py

@@ -0,0 +1,60 @@
+import time
+
+def RetryMark(inRDPSessionKeyStr, inGSettings):
+    """
+    Set mark that Orch will try to reconnect to RDP
+
+    :param inRDPSessionKeyStr: RDP Session key string - to monitor retry count by the RDP Session key
+    :param inGSettings: Orchestrator global settings dict (singleton)
+    :return: None
+
+    """
+
+    lL = inGSettings.get("Logger", None) # Get the logger instance
+    # Create List by the RDP Session key if not exists
+    if inRDPSessionKeyStr not in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]:
+        inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr] = []
+    inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr].append(time.time())
+
+def RetryIsTriggered(inRDPSessionKeyStr, inGSettings):
+    """
+    Check if you can need to init recovery mode for the RDP
+
+    :param inRDPSessionKeyStr: RDP Session key string - to monitor retry count by the RDP Session key
+    :param inGSettings: Orchestrator global settings dict (singleton)
+    :return: True - Ready to start recovery mode - remotely restart PC; Falsew - else case
+    """
+    lTimeNowFloat = time.time()
+    lResultBool = False
+    if inRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]:
+        lTimeNewList = []
+        lCatchCounterInt = 0
+        for inTimeItemFloat in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr]:
+            lTimeDeltaFloat = lTimeNowFloat - inTimeItemFloat
+            # Remove item if very old
+            if lTimeDeltaFloat < inGSettings["RobotRDPActive"]["RecoveryDict"]["CatchPeriodSecFloat"]:
+                lTimeNewList.append(inTimeItemFloat)
+                lCatchCounterInt = lCatchCounterInt+1
+        # Set updated list
+        inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr] = lTimeNewList
+        # Check if counter equal or more
+        if lCatchCounterInt>= inGSettings["RobotRDPActive"]["RecoveryDict"]["TriggerCountInt"]:
+            lResultBool = True
+    return lResultBool
+
+def RetryHostClear(inHostStr, inGSettings):
+    """
+    Clear retry stat by the host str
+
+    :param inHostStr: PC host str to be cleared (search the RDPSession keys)
+    :param inGSettings: Orchestrator global settings dict (singleton)
+    :return:
+    """
+
+    for lRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RDPList"]:
+        lRDPDict = inGSettings["RobotRDPActive"]["RDPList"][lRDPSessionKeyStr]
+        # Check if HOST in UPPER is equal
+        if lRDPDict["Host"].upper() == inHostStr.upper():
+            #Check if RDPSession key exist in stat
+            if lRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]:
+                del inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][lRDPSessionKeyStr]

Sources/pyOpenRPA/Orchestrator/RobotRDPActive/RobotRDPActive.py

@@ -4,6 +4,8 @@ import time # Time wait operations
 from . import ConnectorExceptions # Exceptions classes
 from . import Connector
 from . import Processor # Module for process some functions on thr RDP
+from . import Recovery
+from .. import __Orchestrator__
 # Main function
 # inThreadControlDict = {"ThreadExecuteBool":True}
 def RobotRDPActive(inGSettings, inThreadControlDict):
@@ -42,14 +44,17 @@ def RobotRDPActive(inGSettings, inThreadControlDict):
                     # Check RDP window is OK - reconnect if connection was lost
                     lUIOSelectorList = []
                     lRDPConfigurationDictList = []
+                    lRDPSessionKeyList = []
                     # Prepare selectors list for check
                     for lRDPSessionKeyStrItem in inGlobalDict["RDPList"]:
+                        lRDPSessionKeyList.append(lRDPSessionKeyStrItem)
                         lItem = inGlobalDict["RDPList"][lRDPSessionKeyStrItem]
                         lRDPConfigurationDictList.append(lItem) # Add RDP Configuration in list
                         lUIOSelectorList.append([{"title_re": f"{lItem['SessionHex']}.*", "backend": "win32"}])
                     # Run wait command
                     lRDPDissappearList = UIDesktop.UIOSelectorsSecs_WaitDisappear_List(lUIOSelectorList, inListUpdateTimeout)
                     for lItem in lRDPDissappearList: # Reconnect if connection was lost
+                        lRDPSessionKeyStr = lRDPSessionKeyList[lItem]
                         lRDPConfigurationDict = lRDPConfigurationDictList[lItem] # Get RDP Configuration list
                         lRDPConfigurationDict["SessionIsWindowExistBool"] = False  # Set flag that session is disconnected
                         # Check if RDP window is not ignored
@@ -57,11 +62,19 @@ def RobotRDPActive(inGSettings, inThreadControlDict):
                             try:
                                 Connector.Session(lRDPConfigurationDict, inScreenSize550x350Bool = True)
                                 lRDPConfigurationDict["SessionIsWindowExistBool"] = True  # Flag that session is started
-                                if lL: lL.info(f"SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session has been initialized!")  #Logging
+                                if lL: lL.info(f"Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}, SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session has been initialized!")  #Logging
                             # catch ConnectorExceptions.SessionWindowNotExistError
                             except ConnectorExceptions.SessionWindowNotExistError as e:
                                 lRDPConfigurationDict["SessionIsWindowExistBool"] = False  # Set flag that session is disconnected
-                                if lL: lL.warning(f"SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session is not exist!")  #Logging
+                                if lL: lL.warning(f"Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}, SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session is not exist! Mark the retry")  #Logging
+                                # Recovery operations
+                                Recovery.RetryMark(inRDPSessionKeyStr=lRDPSessionKeyStr,inGSettings=inGSettings)
+                                if Recovery.RetryIsTriggered(inRDPSessionKeyStr=lRDPSessionKeyStr,inGSettings=inGSettings) == True:
+                                    if lL: lL.warning(f"!ATTENTION! Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}; RDP is not responsible for many times - run recovery mode")
+                                    Recovery.RetryHostClear(inHostStr=lRDPConfigurationDict['Host'],inGSettings=inGSettings) # Clear the stat about current host
+                                    if inGSettings["RobotRDPActive"]["RecoveryDict"]["DoDict"]["OSRemotePCRestart"] == True:
+                                        if lL: lL.warning(f"!ATTENTION! Host: {lRDPConfigurationDict['Host']}, Send signal to restart remote PC.")
+                                        __Orchestrator__.OSRemotePCRestart(inLogger=lL,inHostStr=lRDPConfigurationDict['Host'],inForceBool=True)
                             # general exceptions
                             except Exception as e:
                                 if lL: lL.exception(f"!!! ATTENTION !!! Unrecognized error")  #Logging

Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py

@@ -172,6 +172,16 @@ def __Create__():
         },
         # # # # # # # # # # # # # #
         "RobotRDPActive": {
+            "RecoveryDict": {
+                "CatchPeriodSecFloat": 1200, # Catch last 10 minutes
+                "TriggerCountInt": 10, # Activate trigger if for the period orch will catch the reconnect RDP n times
+                "DoDict": {
+                    "OSRemotePCRestart": True # Do powershell remote restart
+                },
+                "__StatisticsDict__": {
+                    # RDPSessionKeyStr : [time.time(), time.time()],
+                }
+            },
             "RDPList": {
                 # "RDPSessionKey":{
                 #    "Host": "77.77.22.22",  # Host address

Sources/pyOpenRPA/Orchestrator/__Orchestrator__.py

@@ -270,6 +270,20 @@ def OSCredentialsVerify(inUserStr, inPasswordStr, inDomainStr=""): ##
     else:
         return True
 
+def OSRemotePCRestart(inLogger, inHostStr, inForceBool=True):
+    """
+    Send signal via power shell to restart remote PC
+    ATTENTION: Orchestrator user need to have restart right on the Remote machine to restart PC.
+
+    :param inLogger: logger to log powershell result in logs
+    :param inHostStr: PC hostname which you need to restart.
+    :param inForceBool: True - send signal to force retart PC; False - else case
+    :return:
+    """
+    lCMDStr = f"powershell -Command Restart-Computer -ComputerName {inHostStr}"
+    if inForceBool == True: lCMDStr = lCMDStr + " -Force"
+    OSCMD(inCMDStr=lCMDStr,inLogger=inLogger)
+
 def OSCMD(inCMDStr, inRunAsyncBool=True, inLogger = None):
     """
     OS send command in shell locally