Add RecoveryMode for the RDP Active

dev-linux
Ivan Maslov 4 years ago
parent df357e58ce
commit 938e36f8a7

@ -441,3 +441,17 @@ def Update(inGSettings):
inGSettings["ServerDict"]["AgentLoopSleepSecFloat"] = 2.0 inGSettings["ServerDict"]["AgentLoopSleepSecFloat"] = 2.0
if lL: lL.warning( if lL: lL.warning(
f"Backward compatibility (v1.2.1 to v1.2.2): Add key AgentActivityLifetimeSecFloat, AgentConnectionLifetimeSecFloat, AgentLoopSleepSecFloat in ProcessorDict > ServerDict") # Log about compatibility f"Backward compatibility (v1.2.1 to v1.2.2): Add key AgentActivityLifetimeSecFloat, AgentConnectionLifetimeSecFloat, AgentLoopSleepSecFloat in ProcessorDict > ServerDict") # Log about compatibility
# Add new key RecoveryDict in ProcessorDict > RobotRDPActive
if "RecoveryDict" not in inGSettings["RobotRDPActive"]:
inGSettings["RobotRDPActive"]["RecoveryDict"] = {
"CatchPeriodSecFloat": 1200, # Catch last 10 minutes
"TriggerCountInt": 10, # Activate trigger if for the period orch will catch the reconnect RDP n times
"DoDict": {
"OSRemotePCRestart": True # Do powershell remote restart
},
"__StatisticsDict__": {
# RDPSessionKeyStr : [time.time(), time.time()],
}
}
if lL: lL.warning(
f"Backward compatibility (v1.2.1 to v1.2.2): Add new key RecoveryDict in ProcessorDict > RobotRDPActive") # Log about compatibility

@ -0,0 +1,60 @@
import time
def RetryMark(inRDPSessionKeyStr, inGSettings):
"""
Set mark that Orch will try to reconnect to RDP
:param inRDPSessionKeyStr: RDP Session key string - to monitor retry count by the RDP Session key
:param inGSettings: Orchestrator global settings dict (singleton)
:return: None
"""
lL = inGSettings.get("Logger", None) # Get the logger instance
# Create List by the RDP Session key if not exists
if inRDPSessionKeyStr not in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]:
inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr] = []
inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr].append(time.time())
def RetryIsTriggered(inRDPSessionKeyStr, inGSettings):
"""
Check if you can need to init recovery mode for the RDP
:param inRDPSessionKeyStr: RDP Session key string - to monitor retry count by the RDP Session key
:param inGSettings: Orchestrator global settings dict (singleton)
:return: True - Ready to start recovery mode - remotely restart PC; Falsew - else case
"""
lTimeNowFloat = time.time()
lResultBool = False
if inRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]:
lTimeNewList = []
lCatchCounterInt = 0
for inTimeItemFloat in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr]:
lTimeDeltaFloat = lTimeNowFloat - inTimeItemFloat
# Remove item if very old
if lTimeDeltaFloat < inGSettings["RobotRDPActive"]["RecoveryDict"]["CatchPeriodSecFloat"]:
lTimeNewList.append(inTimeItemFloat)
lCatchCounterInt = lCatchCounterInt+1
# Set updated list
inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][inRDPSessionKeyStr] = lTimeNewList
# Check if counter equal or more
if lCatchCounterInt>= inGSettings["RobotRDPActive"]["RecoveryDict"]["TriggerCountInt"]:
lResultBool = True
return lResultBool
def RetryHostClear(inHostStr, inGSettings):
"""
Clear retry stat by the host str
:param inHostStr: PC host str to be cleared (search the RDPSession keys)
:param inGSettings: Orchestrator global settings dict (singleton)
:return:
"""
for lRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RDPList"]:
lRDPDict = inGSettings["RobotRDPActive"]["RDPList"][lRDPSessionKeyStr]
# Check if HOST in UPPER is equal
if lRDPDict["Host"].upper() == inHostStr.upper():
#Check if RDPSession key exist in stat
if lRDPSessionKeyStr in inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"]:
del inGSettings["RobotRDPActive"]["RecoveryDict"]["__StatisticsDict__"][lRDPSessionKeyStr]

@ -4,6 +4,8 @@ import time # Time wait operations
from . import ConnectorExceptions # Exceptions classes from . import ConnectorExceptions # Exceptions classes
from . import Connector from . import Connector
from . import Processor # Module for process some functions on thr RDP from . import Processor # Module for process some functions on thr RDP
from . import Recovery
from .. import __Orchestrator__
# Main function # Main function
# inThreadControlDict = {"ThreadExecuteBool":True} # inThreadControlDict = {"ThreadExecuteBool":True}
def RobotRDPActive(inGSettings, inThreadControlDict): def RobotRDPActive(inGSettings, inThreadControlDict):
@ -42,14 +44,17 @@ def RobotRDPActive(inGSettings, inThreadControlDict):
# Check RDP window is OK - reconnect if connection was lost # Check RDP window is OK - reconnect if connection was lost
lUIOSelectorList = [] lUIOSelectorList = []
lRDPConfigurationDictList = [] lRDPConfigurationDictList = []
lRDPSessionKeyList = []
# Prepare selectors list for check # Prepare selectors list for check
for lRDPSessionKeyStrItem in inGlobalDict["RDPList"]: for lRDPSessionKeyStrItem in inGlobalDict["RDPList"]:
lRDPSessionKeyList.append(lRDPSessionKeyStrItem)
lItem = inGlobalDict["RDPList"][lRDPSessionKeyStrItem] lItem = inGlobalDict["RDPList"][lRDPSessionKeyStrItem]
lRDPConfigurationDictList.append(lItem) # Add RDP Configuration in list lRDPConfigurationDictList.append(lItem) # Add RDP Configuration in list
lUIOSelectorList.append([{"title_re": f"{lItem['SessionHex']}.*", "backend": "win32"}]) lUIOSelectorList.append([{"title_re": f"{lItem['SessionHex']}.*", "backend": "win32"}])
# Run wait command # Run wait command
lRDPDissappearList = UIDesktop.UIOSelectorsSecs_WaitDisappear_List(lUIOSelectorList, inListUpdateTimeout) lRDPDissappearList = UIDesktop.UIOSelectorsSecs_WaitDisappear_List(lUIOSelectorList, inListUpdateTimeout)
for lItem in lRDPDissappearList: # Reconnect if connection was lost for lItem in lRDPDissappearList: # Reconnect if connection was lost
lRDPSessionKeyStr = lRDPSessionKeyList[lItem]
lRDPConfigurationDict = lRDPConfigurationDictList[lItem] # Get RDP Configuration list lRDPConfigurationDict = lRDPConfigurationDictList[lItem] # Get RDP Configuration list
lRDPConfigurationDict["SessionIsWindowExistBool"] = False # Set flag that session is disconnected lRDPConfigurationDict["SessionIsWindowExistBool"] = False # Set flag that session is disconnected
# Check if RDP window is not ignored # Check if RDP window is not ignored
@ -57,11 +62,19 @@ def RobotRDPActive(inGSettings, inThreadControlDict):
try: try:
Connector.Session(lRDPConfigurationDict, inScreenSize550x350Bool = True) Connector.Session(lRDPConfigurationDict, inScreenSize550x350Bool = True)
lRDPConfigurationDict["SessionIsWindowExistBool"] = True # Flag that session is started lRDPConfigurationDict["SessionIsWindowExistBool"] = True # Flag that session is started
if lL: lL.info(f"SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session has been initialized!") #Logging if lL: lL.info(f"Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}, SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session has been initialized!") #Logging
# catch ConnectorExceptions.SessionWindowNotExistError # catch ConnectorExceptions.SessionWindowNotExistError
except ConnectorExceptions.SessionWindowNotExistError as e: except ConnectorExceptions.SessionWindowNotExistError as e:
lRDPConfigurationDict["SessionIsWindowExistBool"] = False # Set flag that session is disconnected lRDPConfigurationDict["SessionIsWindowExistBool"] = False # Set flag that session is disconnected
if lL: lL.warning(f"SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session is not exist!") #Logging if lL: lL.warning(f"Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}, SessionHex: {str(lRDPConfigurationDict['SessionHex'])}:: Session is not exist! Mark the retry") #Logging
# Recovery operations
Recovery.RetryMark(inRDPSessionKeyStr=lRDPSessionKeyStr,inGSettings=inGSettings)
if Recovery.RetryIsTriggered(inRDPSessionKeyStr=lRDPSessionKeyStr,inGSettings=inGSettings) == True:
if lL: lL.warning(f"!ATTENTION! Host: {lRDPConfigurationDict['Host']}, Login: {lRDPConfigurationDict['Login']}; RDP is not responsible for many times - run recovery mode")
Recovery.RetryHostClear(inHostStr=lRDPConfigurationDict['Host'],inGSettings=inGSettings) # Clear the stat about current host
if inGSettings["RobotRDPActive"]["RecoveryDict"]["DoDict"]["OSRemotePCRestart"] == True:
if lL: lL.warning(f"!ATTENTION! Host: {lRDPConfigurationDict['Host']}, Send signal to restart remote PC.")
__Orchestrator__.OSRemotePCRestart(inLogger=lL,inHostStr=lRDPConfigurationDict['Host'],inForceBool=True)
# general exceptions # general exceptions
except Exception as e: except Exception as e:
if lL: lL.exception(f"!!! ATTENTION !!! Unrecognized error") #Logging if lL: lL.exception(f"!!! ATTENTION !!! Unrecognized error") #Logging

@ -172,6 +172,16 @@ def __Create__():
}, },
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
"RobotRDPActive": { "RobotRDPActive": {
"RecoveryDict": {
"CatchPeriodSecFloat": 1200, # Catch last 10 minutes
"TriggerCountInt": 10, # Activate trigger if for the period orch will catch the reconnect RDP n times
"DoDict": {
"OSRemotePCRestart": True # Do powershell remote restart
},
"__StatisticsDict__": {
# RDPSessionKeyStr : [time.time(), time.time()],
}
},
"RDPList": { "RDPList": {
# "RDPSessionKey":{ # "RDPSessionKey":{
# "Host": "77.77.22.22", # Host address # "Host": "77.77.22.22", # Host address

@ -270,6 +270,20 @@ def OSCredentialsVerify(inUserStr, inPasswordStr, inDomainStr=""): ##
else: else:
return True return True
def OSRemotePCRestart(inLogger, inHostStr, inForceBool=True):
"""
Send signal via power shell to restart remote PC
ATTENTION: Orchestrator user need to have restart right on the Remote machine to restart PC.
:param inLogger: logger to log powershell result in logs
:param inHostStr: PC hostname which you need to restart.
:param inForceBool: True - send signal to force retart PC; False - else case
:return:
"""
lCMDStr = f"powershell -Command Restart-Computer -ComputerName {inHostStr}"
if inForceBool == True: lCMDStr = lCMDStr + " -Force"
OSCMD(inCMDStr=lCMDStr,inLogger=inLogger)
def OSCMD(inCMDStr, inRunAsyncBool=True, inLogger = None): def OSCMD(inCMDStr, inRunAsyncBool=True, inLogger = None):
""" """
OS send command in shell locally OS send command in shell locally

Loading…
Cancel
Save