diff --git a/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py b/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py index d853692e..6c340444 100644 --- a/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py +++ b/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py @@ -441,3 +441,17 @@ def Update(inGSettings): inGSettings["ServerDict"]["AgentLoopSleepSecFloat"] = 2.0 if lL: lL.warning( f"Backward compatibility (v1.2.1 to v1.2.2): Add key AgentActivityLifetimeSecFloat, AgentConnectionLifetimeSecFloat, AgentLoopSleepSecFloat in ProcessorDict > ServerDict") # Log about compatibility + # Add new key RecoveryDict in ProcessorDict > RobotRDPActive + if "RecoveryDict" not in inGSettings["RobotRDPActive"]: + inGSettings["RobotRDPActive"]["RecoveryDict"] = { + "CatchPeriodSecFloat": 1200, # Catch last 10 minutes + "TriggerCountInt": 10, # Activate trigger if for the period orch will catch the reconnect RDP n times + "DoDict": { + "OSRemotePCRestart": True # Do powershell remote restart + }, + "__StatisticsDict__": { + # RDPSessionKeyStr : [time.time(), time.time()], + } + } + if lL: lL.warning( + f"Backward compatibility (v1.2.1 to v1.2.2): Add new key RecoveryDict in ProcessorDict > RobotRDPActive") # Log about compatibility diff --git a/Sources/pyOpenRPA/Orchestrator/RobotRDPActive/Recovery.py b/Sources/pyOpenRPA/Orchestrator/RobotRDPActive/Recovery.py new file mode 100644 index 00000000..2397c33a --- /dev/null +++ b/Sources/pyOpenRPA/Orchestrator/RobotRDPActive/Recovery.py @@ -0,0 +1,60 @@ +import time + +def RetryMark(inRDPSessionKeyStr, inGSettings): + """ + Set mark that Orch will try to reconnect to RDP + + :param inRDPSessionKeyStr: RDP Session key string - to monitor retry count by the RDP Session key + :param inGSettings: Orchestrator global settings dict (singleton) + :return: None + + """ + + lL = inGSettings.get("Logger", None) # Get the logger instance + # Create List by the RDP Session key if not exists + if inRDPSessionKeyStr not in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]: + inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr] = [] + inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr].append(time.time()) + +def RetryIsTriggered(inRDPSessionKeyStr, inGSettings): + """ + Check if you can need to init recovery mode for the RDP + + :param inRDPSessionKeyStr: RDP Session key string - to monitor retry count by the RDP Session key + :param inGSettings: Orchestrator global settings dict (singleton) + :return: True - Ready to start recovery mode - remotely restart PC; Falsew - else case + """ + lTimeNowFloat = time.time() + lResultBool = False + if inRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]: + lTimeNewList = [] + lCatchCounterInt = 0 + for inTimeItemFloat in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr]: + lTimeDeltaFloat = lTimeNowFloat - inTimeItemFloat + # Remove item if very old + if lTimeDeltaFloat < inGSettings["RobotRDPActive"]["RecoveryDict"]["CatchPeriodSecFloat"]: + lTimeNewList.append(inTimeItemFloat) + lCatchCounterInt = lCatchCounterInt+1 + # Set updated list + inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr] = lTimeNewList + # Check if counter equal or more + if lCatchCounterInt>= inGSettings["RobotRDPActive"]["RecoveryDict"]["TriggerCountInt"]: + lResultBool = True + return lResultBool + +def RetryHostClear(inHostStr, inGSettings): + """ + Clear retry stat by the host str + + :param inHostStr: PC host str to be cleared (search the RDPSession keys) + :param inGSettings: Orchestrator global settings dict (singleton) + :return: + """ + + for lRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RDPList"]: + lRDPDict = inGSettings["RobotRDPActive"]["RDPList"][lRDPSessionKeyStr] + # Check if HOST in UPPER is equal + if lRDPDict["Host"].upper() == inHostStr.upper(): + #Check if RDPSession key exist in stat + if lRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]: + del inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][lRDPSessionKeyStr] \ No newline at end of file diff --git a/Sources/pyOpenRPA/Orchestrator/RobotRDPActive/RobotRDPActive.py b/Sources/pyOpenRPA/Orchestrator/RobotRDPActive/RobotRDPActive.py index e0707bb1..60b6631e 100644 --- a/Sources/pyOpenRPA/Orchestrator/RobotRDPActive/RobotRDPActive.py +++ b/Sources/pyOpenRPA/Orchestrator/RobotRDPActive/RobotRDPActive.py @@ -4,6 +4,8 @@ import time # Time wait operations from . import ConnectorExceptions # Exceptions classes from . import Connector from . import Processor # Module for process some functions on thr RDP +from . import Recovery +from .. import __Orchestrator__ # Main function # inThreadControlDict = {"ThreadExecuteBool":True} def RobotRDPActive(inGSettings, inThreadControlDict): @@ -42,14 +44,17 @@ def RobotRDPActive(inGSettings, inThreadControlDict): # Check RDP window is OK - reconnect if connection was lost lUIOSelectorList = [] lRDPConfigurationDictList = [] + lRDPSessionKeyList = [] # Prepare selectors list for check for lRDPSessionKeyStrItem in inGlobalDict["RDPList"]: + lRDPSessionKeyList.append(lRDPSessionKeyStrItem) lItem = inGlobalDict["RDPList"][lRDPSessionKeyStrItem] lRDPConfigurationDictList.append(lItem) # Add RDP Configuration in list lUIOSelectorList.append([{"title_re": f"{lItem['SessionHex']}.*", "backend": "win32"}]) # Run wait command lRDPDissappearList = UIDesktop.UIOSelectorsSecs_WaitDisappear_List(lUIOSelectorList, inListUpdateTimeout) for lItem in lRDPDissappearList: # Reconnect if connection was lost + lRDPSessionKeyStr = lRDPSessionKeyList[lItem] lRDPConfigurationDict = lRDPConfigurationDictList[lItem] # Get RDP Configuration list lRDPConfigurationDict["SessionIsWindowExistBool"] = False # Set flag that session is disconnected # Check if RDP window is not ignored @@ -57,11 +62,19 @@ def RobotRDPActive(inGSettings, inThreadControlDict): try: Connector.Session(lRDPConfigurationDict, inScreenSize550x350Bool = True) lRDPConfigurationDict["SessionIsWindowExistBool"] = True # Flag that session is started - if lL: lL.info(f"SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session has been initialized!") #Logging + if lL: lL.info(f"Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}, SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session has been initialized!") #Logging # catch ConnectorExceptions.SessionWindowNotExistError except ConnectorExceptions.SessionWindowNotExistError as e: lRDPConfigurationDict["SessionIsWindowExistBool"] = False # Set flag that session is disconnected - if lL: lL.warning(f"SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session is not exist!") #Logging + if lL: lL.warning(f"Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}, SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session is not exist! Mark the retry") #Logging + # Recovery operations + Recovery.RetryMark(inRDPSessionKeyStr=lRDPSessionKeyStr,inGSettings=inGSettings) + if Recovery.RetryIsTriggered(inRDPSessionKeyStr=lRDPSessionKeyStr,inGSettings=inGSettings) == True: + if lL: lL.warning(f"!ATTENTION! Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}; RDP is not responsible for many times - run recovery mode") + Recovery.RetryHostClear(inHostStr=lRDPConfigurationDict['Host'],inGSettings=inGSettings) # Clear the stat about current host + if inGSettings["RobotRDPActive"]["RecoveryDict"]["DoDict"]["OSRemotePCRestart"] == True: + if lL: lL.warning(f"!ATTENTION! Host: {lRDPConfigurationDict['Host']}, Send signal to restart remote PC.") + __Orchestrator__.OSRemotePCRestart(inLogger=lL,inHostStr=lRDPConfigurationDict['Host'],inForceBool=True) # general exceptions except Exception as e: if lL: lL.exception(f"!!! ATTENTION !!! Unrecognized error") #Logging diff --git a/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py b/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py index be224a7d..10570556 100644 --- a/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py +++ b/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py @@ -172,6 +172,16 @@ def __Create__(): }, # # # # # # # # # # # # # # "RobotRDPActive": { + "RecoveryDict": { + "CatchPeriodSecFloat": 1200, # Catch last 10 minutes + "TriggerCountInt": 10, # Activate trigger if for the period orch will catch the reconnect RDP n times + "DoDict": { + "OSRemotePCRestart": True # Do powershell remote restart + }, + "__StatisticsDict__": { + # RDPSessionKeyStr : [time.time(), time.time()], + } + }, "RDPList": { # "RDPSessionKey":{ # "Host": "77.77.22.22", # Host address diff --git a/Sources/pyOpenRPA/Orchestrator/__Orchestrator__.py b/Sources/pyOpenRPA/Orchestrator/__Orchestrator__.py index 35f2abe8..1fcf5d9a 100644 --- a/Sources/pyOpenRPA/Orchestrator/__Orchestrator__.py +++ b/Sources/pyOpenRPA/Orchestrator/__Orchestrator__.py @@ -270,6 +270,20 @@ def OSCredentialsVerify(inUserStr, inPasswordStr, inDomainStr=""): ## else: return True +def OSRemotePCRestart(inLogger, inHostStr, inForceBool=True): + """ + Send signal via power shell to restart remote PC + ATTENTION: Orchestrator user need to have restart right on the Remote machine to restart PC. + + :param inLogger: logger to log powershell result in logs + :param inHostStr: PC hostname which you need to restart. + :param inForceBool: True - send signal to force retart PC; False - else case + :return: + """ + lCMDStr = f"powershell -Command Restart-Computer -ComputerName {inHostStr}" + if inForceBool == True: lCMDStr = lCMDStr + " -Force" + OSCMD(inCMDStr=lCMDStr,inLogger=inLogger) + def OSCMD(inCMDStr, inRunAsyncBool=True, inLogger = None): """ OS send command in shell locally