From 913505ae0e3a808c7db84a92b802a650f3169a22 Mon Sep 17 00:00:00 2001 From: Ivan Maslov Date: Sun, 6 Dec 2020 15:25:28 +0300 Subject: [PATCH] - - def AgentOSFileDataBytesCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileDataBytes): # Send to agent activity item to OSCMD - - def AgentOSFileDataStrCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileDataStr, inEncodingStr="utf-8"): # Send to agent activity item to OSCMD Need to test Agent.FileCreate - has ploblem with transmit bytes on JSON - json dont support bytes string... Need safe encode/decode --- Agent/AgentDaemonX64Debug.cmd | 4 ++++ Agent/AgentSettings.py | 2 +- Sources/pyOpenRPA/Agent/Agent.py | 10 +++++----- Sources/pyOpenRPA/Agent/O2A.py | 4 ++++ Sources/pyOpenRPA/Agent/Processor.py | 10 +++++++--- .../pyOpenRPA/Orchestrator/Orchestrator.py | 20 ++++++++++++------- Sources/pyOpenRPA/Orchestrator/Processor.py | 5 +++-- .../pyOpenRPA/Orchestrator/ServerSettings.py | 17 ++++++++++------ .../Orchestrator/SettingsTemplate.py | 2 +- Sources/pyOpenRPA/Orchestrator/Web/Index.js | 1 - changelog.md | 3 ++- 11 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 Agent/AgentDaemonX64Debug.cmd diff --git a/Agent/AgentDaemonX64Debug.cmd b/Agent/AgentDaemonX64Debug.cmd new file mode 100644 index 00000000..b84d5675 --- /dev/null +++ b/Agent/AgentDaemonX64Debug.cmd @@ -0,0 +1,4 @@ +cd %~dp0 +copy /Y ..\Resources\WPy64-3720\python-3.7.2.amd64\python.exe ..\Resources\WPy64-3720\python-3.7.2.amd64\pyOpenRPA_Agent.exe +.\..\Resources\WPy64-3720\python-3.7.2.amd64\pyOpenRPA_Agent.exe "AgentSettings.py" +pause >nul \ No newline at end of file diff --git a/Agent/AgentSettings.py b/Agent/AgentSettings.py index b1efad79..09d30848 100644 --- a/Agent/AgentSettings.py +++ b/Agent/AgentSettings.py @@ -52,7 +52,7 @@ if __name__ == "__main__": # New init way # Подготовка логгера Robot ######################### lL = gSettings["Logger"] - lL.setLevel(logging.DEBUG) + lL.setLevel(logging.INFO) # create the logging file handler mRobotLoggerFH = logging.FileHandler( "Reports\\" + datetime.datetime.now().strftime("%Y_%m_%d") + ".log") diff --git a/Sources/pyOpenRPA/Agent/Agent.py b/Sources/pyOpenRPA/Agent/Agent.py index 65674e67..02f9a4fd 100644 --- a/Sources/pyOpenRPA/Agent/Agent.py +++ b/Sources/pyOpenRPA/Agent/Agent.py @@ -3,11 +3,11 @@ from . import O2A, A2O # Data flow Orchestrator To Agent from . import Processor # Processor Queue # Send CMD to OS. Result return to log + Orchestrator by the A2O connection -def OSFileBytesCreate(inFilePathStr, inFileBytes,inGSettings = None): +def OSFileDataBytesCreate(inFilePathStr, inFileDataBytes,inGSettings = None): lFile = open(inFilePathStr, "wb") - lFile.write(inFileBytes) + lFile.write(inFileDataBytes) lL = inGSettings.get("Logger", None) if type(inGSettings) is dict else None - lMessageStr = f"File {inFilePathStr} has been created." + lMessageStr = f"AGENT File {inFilePathStr} has been created." if lL: lL.info(lMessageStr) A2O.LogListSend(inGSettings=inGSettings, inLogList=[lMessageStr]) @@ -21,7 +21,7 @@ def OSCMD(inCMDStr, inRunAsyncBool=True, inGSettings = None): lOSCMDKeyStr = str(uuid.uuid4())[0:4].upper() lCMDProcess = subprocess.Popen(f'cmd /c {inCMDStr}', stdout=subprocess.PIPE, stderr=subprocess.STDOUT) lListenBool = True - lMessageStr = f"{lOSCMDKeyStr}: # # # # CMD Process has been STARTED # # # # " + lMessageStr = f"{lOSCMDKeyStr}: # # # # AGENT CMD Process has been STARTED # # # # " if lL: lL.info(lMessageStr) A2O.LogListSend(inGSettings=inGSettings,inLogList=[lMessageStr]) lMessageStr = f"{lOSCMDKeyStr}: {inCMDStr}" @@ -37,7 +37,7 @@ def OSCMD(inCMDStr, inRunAsyncBool=True, inGSettings = None): if lL: lL.info(lMessageStr) A2O.LogListSend(inGSettings=inGSettings, inLogList=[lMessageStr]) lResultStr+=lStr - lMessageStr = f"{lOSCMDKeyStr}: # # # # CMD Process has been FINISHED # # # # " + lMessageStr = f"{lOSCMDKeyStr}: # # # # AGENT CMD Process has been FINISHED # # # # " if lL: lL.info(lMessageStr) A2O.LogListSend(inGSettings=inGSettings, inLogList=[lMessageStr]) return lResultStr diff --git a/Sources/pyOpenRPA/Agent/O2A.py b/Sources/pyOpenRPA/Agent/O2A.py index 5ab71cbe..082d7827 100644 --- a/Sources/pyOpenRPA/Agent/O2A.py +++ b/Sources/pyOpenRPA/Agent/O2A.py @@ -32,6 +32,10 @@ def O2A_Loop(inGSettings): lQueueItem = lResponse.json() # Try to get JSON # Append QUEUE item in ProcessorDict > ActivityList inGSettings["ProcessorDict"]["ActivityList"].append(lQueueItem) + if lL: lL.debug(f"ActivityItem was recieved from orchestrator: {lQueueItem}"); except Exception as e: if lL: lL.exception(f"A2O Error handler. Sleep for {inGSettings['A2ODict']['RetryTimeoutSecFloat']} s.") + time.sleep(inGSettings["O2ADict"]["RetryTimeoutSecFloat"]) + except requests.exceptions.ConnectionError as e: + if lL: lL.error(f"A2O Connection error - orchestrator is not available. Sleep for {inGSettings['A2ODict']['RetryTimeoutSecFloat']} s.") time.sleep(inGSettings["O2ADict"]["RetryTimeoutSecFloat"]) \ No newline at end of file diff --git a/Sources/pyOpenRPA/Agent/Processor.py b/Sources/pyOpenRPA/Agent/Processor.py index c254c0f6..302552b2 100644 --- a/Sources/pyOpenRPA/Agent/Processor.py +++ b/Sources/pyOpenRPA/Agent/Processor.py @@ -17,12 +17,16 @@ def ProcessorRunSync(inGSettings): "CheckIntervalSecFloat": 1.0 # Interval for check gSettings in ProcessorDict > ActivityList "ExecuteBool": True # Flag to execute thread processor """ + lL = inGSettings["Logger"] # Logger alias inGSettings["ProcessorDict"]["ThreadIdInt"] = threading.get_ident() # fill Processor thread id while inGSettings["ProcessorDict"]["ExecuteBool"]: - lActivityItem = inGSettings["ProcessorDict"]["ActivityList"].pop(0, None) # Extract the first item from processor queue - while lActivityItem is not None: + lActivityList = inGSettings["ProcessorDict"]["ActivityList"] # Alias + if len(lActivityList)>0: + if lL: lL.debug(f'Processor ActivityList len: {len(lActivityList)}') + lActivityItem = inGSettings["ProcessorDict"]["ActivityList"].pop(0) # Extract the first item from processor queue ActivityListExecute(inGSettings = inGSettings, inActivityList = [lActivityItem]) # execute the activity item - time.sleep(inGSettings["ProcessorDict"]["CheckIntervalSecFloat"]) # Sleep when list is empty + else: + time.sleep(inGSettings["ProcessorDict"]["CheckIntervalSecFloat"]) # Sleep when list is empty # Execute ActivityItem list # return the def result diff --git a/Sources/pyOpenRPA/Orchestrator/Orchestrator.py b/Sources/pyOpenRPA/Orchestrator/Orchestrator.py index 7d779fc4..f9f2b8ad 100644 --- a/Sources/pyOpenRPA/Orchestrator/Orchestrator.py +++ b/Sources/pyOpenRPA/Orchestrator/Orchestrator.py @@ -33,17 +33,17 @@ def AgentActivityItemAdd(inGSettings, inHostNameStr, inUserStr, inActivityItemDi if lAgentDictItemKeyTurple not in inGSettings["AgentDict"]: inGSettings["AgentDict"][lAgentDictItemKeyTurple] = SettingsTemplate.__AgentDictItemCreate__() lThisAgentDict = inGSettings["AgentDict"][lAgentDictItemKeyTurple] - lThisAgentDict["ActivityList"]+=inActivityItemDict + lThisAgentDict["ActivityList"].append(inActivityItemDict) # Send to agent activity item to OSCMD -def AgentOSCMD(inGSettings, inHostNameStr, inUserStr, inCMDStr): +def AgentOSCMD(inGSettings, inHostNameStr, inUserStr, inCMDStr, inRunAsyncBool=True): # pyOpenRPA.Agent: Send CMD to OS. Result return to log + Orchestrator by the A2O connection # def OSCMD(inCMDStr, inRunAsyncBool=True, inGSettings=None): # Create Activity Item for the agent lActivityItemDict = { "Def":"OSCMD", # def alias (look pyOpeRPA.Agent gSettings["ProcessorDict"]["AliasDefDict"]) "ArgList":[], # Args list - "ArgDict":{"inCMDStr":inCMDStr,"inRunAsyncBool":False}, # Args dictionary + "ArgDict":{"inCMDStr":inCMDStr,"inRunAsyncBool":inRunAsyncBool}, # Args dictionary "ArgGSettings": "inGSettings", # Name of GSettings attribute: str (ArgDict) or index (for ArgList) "ArgLogger": None # Name of GSettings attribute: str (ArgDict) or index (for ArgList) } @@ -51,20 +51,26 @@ def AgentOSCMD(inGSettings, inHostNameStr, inUserStr, inCMDStr): AgentActivityItemAdd(inGSettings=inGSettings, inHostNameStr=inHostNameStr, inUserStr=inUserStr, inActivityItemDict=lActivityItemDict) # Send to agent activity item to OSCMD -def AgentOSFileBytesCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileBytes): +def AgentOSFileDataBytesCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileDataBytes): # pyOpenRPA.Agent: Send CMD to OS. Result return to log + Orchestrator by the A2O connection - # def OSFileBytesCreate(inFilePathStr, inFileBytes,inGSettings = None): + # def OSFileBytesCreate(inFilePathStr, inFileDataBytes,inGSettings = None): # Create Activity Item for the agent lActivityItemDict = { - "Def":"OSFileBytesCreate", # def alias (look pyOpeRPA.Agent gSettings["ProcessorDict"]["AliasDefDict"]) + "Def":"OSFileDataBytesCreate", # def alias (look pyOpeRPA.Agent gSettings["ProcessorDict"]["AliasDefDict"]) "ArgList":[], # Args list - "ArgDict":{"inFilePathStr":inFilePathStr,"inFileBytes":inFileBytes}, # Args dictionary + "ArgDict":{"inFilePathStr":inFilePathStr,"inFileDataBytes":inFileDataBytes}, # Args dictionary "ArgGSettings": "inGSettings", # Name of GSettings attribute: str (ArgDict) or index (for ArgList) "ArgLogger": None # Name of GSettings attribute: str (ArgDict) or index (for ArgList) } #Send item in AgentDict for the futher data transmition AgentActivityItemAdd(inGSettings=inGSettings, inHostNameStr=inHostNameStr, inUserStr=inUserStr, inActivityItemDict=lActivityItemDict) +# Send to agent activity item to OSCMD +def AgentOSFileDataStrCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileDataStr, inEncodingStr="utf-8"): + # pyOpenRPA.Agent: Send CMD to OS. Result return to log + Orchestrator by the A2O connection + # def OSFileBytesCreate(inFilePathStr, inFileDataBytes,inGSettings = None): + # Create Activity Item for the agent + AgentOSFileDataBytesCreate(inGSettings=inGSettings, inHostNameStr=inHostNameStr, inUserStr=inUserStr, inFilePathStr=inFilePathStr, inFileDataBytes=inFileDataStr.encode(inEncodingStr)) # OS DEFS # Defs to use in orchestrator diff --git a/Sources/pyOpenRPA/Orchestrator/Processor.py b/Sources/pyOpenRPA/Orchestrator/Processor.py index 6d7207e6..512a5280 100644 --- a/Sources/pyOpenRPA/Orchestrator/Processor.py +++ b/Sources/pyOpenRPA/Orchestrator/Processor.py @@ -17,13 +17,14 @@ def ProcessorRunSync(inGSettings): "CheckIntervalSecFloat": 1.0 # Interval for check gSettings in ProcessorDict > ActivityList "ExecuteBool": True # Flag to execute thread processor """ + lL = inGSettings["Logger"] # Logger alias inGSettings["ProcessorDict"]["ThreadIdInt"] = threading.get_ident() # fill Processor thread id while inGSettings["ProcessorDict"]["ExecuteBool"]: lActivityList = inGSettings["ProcessorDict"]["ActivityList"] # Alias if len(lActivityList)>0: + if lL: lL.debug(f'Processor ActivityList len: {len(lActivityList)}') lActivityItem = inGSettings["ProcessorDict"]["ActivityList"].pop(0) # Extract the first item from processor queue - while lActivityItem is not None: - ActivityListExecute(inGSettings = inGSettings, inActivityList = [lActivityItem]) # execute the activity item + ActivityListExecute(inGSettings = inGSettings, inActivityList = [lActivityItem]) # execute the activity item time.sleep(inGSettings["ProcessorDict"]["CheckIntervalSecFloat"]) # Sleep when list is empty # Execute ActivityItem list diff --git a/Sources/pyOpenRPA/Orchestrator/ServerSettings.py b/Sources/pyOpenRPA/Orchestrator/ServerSettings.py index 8db82e6b..6363ace5 100644 --- a/Sources/pyOpenRPA/Orchestrator/ServerSettings.py +++ b/Sources/pyOpenRPA/Orchestrator/ServerSettings.py @@ -212,7 +212,8 @@ def pyOpenRPA_ActivityListExecute(inRequest, inGSettings): # See docs in Agent (pyOpenRPA.Agent.O2A) def pyOpenRPA_Agent_O2A(inRequest, inGSettings): - lConnectionLifetimeSecFloat = 5.0 # 60 min * 60 sec + lL = inGSettings["Logger"] # Alias + lConnectionLifetimeSecFloat = 3600.0 # 60 min * 60 sec 3600.0 lTimeStartFloat = time.time() # Recieve the data lValueStr = None @@ -234,25 +235,29 @@ def pyOpenRPA_Agent_O2A(inRequest, inGSettings): # Check if lifetime is over if time.time() - lTimeStartFloat > lConnectionLifetimeSecFloat: # Lifetime is over lThisAgentDict["IsListenBool"] = False # Set is offline - lThisAgentDict["ConnectionCountInt"] -= 1 # decrement connection count lDoLoopBool = False else: # Lifetime is good - do alg lThisAgentDict["IsListenBool"] = True # Set is online lQueueList = lThisAgentDict["ActivityList"] if len(lQueueList)>0:# Do some operations if has queue items - if lThisAgentDict["ConnectionCountInt"] == lThisAgentDict["ConnectionFirstQueueItemCountInt"] - 1: - # POP QUEUE ITEM CONDITION ConnectionCountInt == ConnectionFirstQueueItemCountInt - 1 + if lL: lL.debug(f'O2A BEFORE: ConnectionCountInt: {lThisAgentDict["ConnectionCountInt"]};ConnectionFirstQueueItemCountInt {lThisAgentDict["ConnectionFirstQueueItemCountInt"]}') + if lThisAgentDict["ConnectionCountInt"] == lThisAgentDict["ConnectionFirstQueueItemCountInt"] + 1: + # POP QUEUE ITEM CONDITION ConnectionCountInt == ConnectionFirstQueueItemCountInt + 1 lQueueItem = lThisAgentDict["ActivityList"].pop(0) lThisAgentDict["ConnectionFirstQueueItemCountInt"] = 0 + if lL: lL.debug(f"Activity was deleted from the list: {lThisAgentDict['ActivityList']}") else: lQueueItem = lThisAgentDict["ActivityList"][0] lThisAgentDict["ConnectionFirstQueueItemCountInt"] += 1 + if lL: lL.debug(f"Activity was !not! deleted from the list: {lThisAgentDict['ActivityList']}") + if lL: lL.debug(f'O2A AFTER: ConnectionCountInt: {lThisAgentDict["ConnectionCountInt"]};ConnectionFirstQueueItemCountInt {lThisAgentDict["ConnectionFirstQueueItemCountInt"]}') # Send QUEUE ITEM + if lL: lL.debug(f"Activity item to agent: {lQueueItem}") inRequest.OpenRPAResponseDict["Body"] = bytes(json.dumps(lQueueItem), "utf8") - lThisAgentDict["ConnectionCountInt"] -= 1 # Connection go to be closed - decrement the connection count + lDoLoopBool = False # CLose the connection else: # no queue item - sleep for the next iteration time.sleep(1) - + lThisAgentDict["ConnectionCountInt"] -= 1 # Connection go to be closed - decrement the connection count # See docs in Agent (pyOpenRPA.Agent.A2O) def pyOpenRPA_Agent_A2O(inRequest, inGSettings): # Recieve the data diff --git a/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py b/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py index ea85db9b..514ca494 100644 --- a/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py +++ b/Sources/pyOpenRPA/Orchestrator/SettingsTemplate.py @@ -349,7 +349,7 @@ def Create(inModeStr="BASIC"): # Подготовка логгера Robot ######################### mRobotLogger = lResult["Logger"] - mRobotLogger.setLevel(logging.DEBUG) + mRobotLogger.setLevel(logging.INFO) # create the logging file handler mRobotLoggerFH = logging.FileHandler( "Reports\\" + datetime.datetime.now().strftime("%Y_%m_%d") + ".log") diff --git a/Sources/pyOpenRPA/Orchestrator/Web/Index.js b/Sources/pyOpenRPA/Orchestrator/Web/Index.js index a74b359e..c9c29712 100644 --- a/Sources/pyOpenRPA/Orchestrator/Web/Index.js +++ b/Sources/pyOpenRPA/Orchestrator/Web/Index.js @@ -365,7 +365,6 @@ $(document).ready(function() { } ///v 1.2.0 pyOpenRPA - /// Execute ActivityItem mGlobal.pyOpenRPA.ActivityItemExecute=function(inActivityItem) { ///EXAMPLE diff --git a/changelog.md b/changelog.md index 14554134..b4651925 100644 --- a/changelog.md +++ b/changelog.md @@ -65,7 +65,8 @@ - Orc: Add Agent Defs - - def AgentActivityItemAdd(inGSettings, inHostNameStr, inUserStr, inActivityItemDict): # Add activity in AgentDict - - def AgentOSCMD(inGSettings, inHostNameStr, inUserStr, inCMDStr): # Send to agent activity item to OSCMD -- - def AgentOSFileBytesCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileBytes): # Send to agent activity item to OSCMD +- - def AgentOSFileDataBytesCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileDataBytes): # Send to agent activity item to OSCMD +- - def AgentOSFileDataStrCreate(inGSettings, inHostNameStr, inUserStr, inFilePathStr, inFileDataStr, inEncodingStr="utf-8"): # Send to agent activity item to OSCMD - Orc WEB: Create mGlobal.pyOpenRPA.ActivityListExecute({}) to test some activities from the front [1.1.0] After 2 month test prefinal with new improovements (+RobotRDPActive in Orchestrator + Easy ControlPanelTemplate)