From 239a2c6c83ba2522bbea41d80a49d7804b5744fe Mon Sep 17 00:00:00 2001 From: Ivan Maslov Date: Sun, 11 Oct 2020 00:36:45 +0300 Subject: [PATCH] # Test new processor in progress - need new setings template --- Sources/Sandbox/JupyterLab_x64_Run.cmd | 3 + Sources/Sandbox/OrchestratorTestInit.ipynb | 159 +++++++++ .../OrchestratorTestProcessorNew.ipynb | 23 ++ .../Orchestrator/BackwardCompatibility.py | 48 +++ .../pyOpenRPA/Orchestrator/Orchestrator.py | 329 +++++++++--------- Sources/pyOpenRPA/Orchestrator/Processor.py | 4 +- .../pyOpenRPA/Orchestrator/ProcessorOld.py | 13 +- Sources/pyOpenRPA/Orchestrator/Server.py | 7 +- changelog.md | 1 + 9 files changed, 419 insertions(+), 168 deletions(-) create mode 100644 Sources/Sandbox/JupyterLab_x64_Run.cmd create mode 100644 Sources/Sandbox/OrchestratorTestInit.ipynb create mode 100644 Sources/Sandbox/OrchestratorTestProcessorNew.ipynb diff --git a/Sources/Sandbox/JupyterLab_x64_Run.cmd b/Sources/Sandbox/JupyterLab_x64_Run.cmd new file mode 100644 index 00000000..0cef7e67 --- /dev/null +++ b/Sources/Sandbox/JupyterLab_x64_Run.cmd @@ -0,0 +1,3 @@ +cd %~dp0 +..\..\Resources\WPy64-3720\python-3.7.2.amd64\python.exe -m jupyterlab +pause >nul \ No newline at end of file diff --git a/Sources/Sandbox/OrchestratorTestInit.ipynb b/Sources/Sandbox/OrchestratorTestInit.ipynb new file mode 100644 index 00000000..7cd24ae4 --- /dev/null +++ b/Sources/Sandbox/OrchestratorTestInit.ipynb @@ -0,0 +1,159 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Init the configuration GSettings" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "lSettingsDirStr = r\"..\\..\\..\\OpenRPA_Creds\\Orchestrator\\Settings\" # Set Gsettings dir path str\n", + "import sys\n", + "sys.path.append(lSettingsDirStr) # Add path with settings in search dir for .py modules\n", + "sys.path.insert(0,r\"..\") # Add path to sources - to import from experimental pyOpenRPA\n", + "import SettingsOrchestratorExample\n", + "gSettings = SettingsOrchestratorExample.Settings()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run the orchestrator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2020-10-11 00:32:27,869 - Orchestrator - INFO - Link the gSettings in submodules\n", + "2020-10-11 00:32:27,878 - Orchestrator - WARNING - Backward compatibility (v1.1.13 to v1.1.14): Add default 'Autocleaner' structure\n", + "2020-10-11 00:32:27,880 - Orchestrator - WARNING - Backward compatibility (v1.1.13 to v1.1.14): Add default 'Client' structure\n", + "2020-10-11 00:32:27,882 - Orchestrator - WARNING - Backward compatibility (v1.1.13 to v1.1.14): Add default 'Server' > 'RequestTimeoutSecFloat' property\n", + "2020-10-11 00:32:27,883 - Orchestrator - WARNING - Backward compatibility (v1.1.13 to v1.1.14): Add default 'OrchestratorStart' > 'DefSettingsUpdatePathList' property list\n", + "2020-10-11 00:32:27,886 - Orchestrator - WARNING - Backward compatibility (v1.1.20 to v1.2.0): Remove old structure 'Processor'\n", + "2020-10-11 00:32:27,888 - Orchestrator - WARNING - Backward compatibility (v1.1.20 to v1.2.0): Create new structure 'ProcessorDict'\n", + "2020-10-11 00:32:27,893 - Orchestrator - INFO - Server init. Listen URL: , Listen port: 80\n", + "2020-10-11 00:32:27,893 - Orchestrator - INFO - Web server has been started\n", + "2020-10-11 00:32:27,932 - Orchestrator - INFO - Robot Screen active has been started\n", + "2020-10-11 00:32:27,937 - Orchestrator - INFO - Robot RDP active has been started\n", + "2020-10-11 00:32:27,940 - Orchestrator - INFO - Autocleaner thread has been started\n", + "2020-10-11 00:32:27,942 - Orchestrator - INFO - Orchestrator start activity run\n", + "2020-10-11 00:32:27,944 - Orchestrator - INFO - Scheduler loop start\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "----------------------------------------\n", + "Exception happened during processing of request from ('127.0.0.1', 57154)\n", + "Traceback (most recent call last):\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\socketserver.py\", line 650, in process_request_thread\n", + " self.finish_request(request, client_address)\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Server.py\", line 476, in finish_request\n", + " HTTPServer.finish_request(self, request, client_address)\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\socketserver.py\", line 360, in finish_request\n", + " self.RequestHandlerClass(request, client_address, self)\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\socketserver.py\", line 720, in __init__\n", + " self.handle()\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\http\\server.py\", line 426, in handle\n", + " self.handle_one_request()\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\http\\server.py\", line 414, in handle_one_request\n", + " method()\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Server.py\", line 336, in do_GET\n", + " lAuthenticateDict = AuthenticateVerify(self)\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Server.py\", line 73, in AuthenticateVerify\n", + " \"Password\": lPassword\n", + " File \"..\\pyOpenRPA\\Orchestrator\\ProcessorOld.py\", line 94, in Activity\n", + " lItem[\"Result\"] = Processor.ActivityListExecute(inGSettings = gSettingsDict, inActivityList = [inActivity])\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Processor.py\", line 32, in ActivityListExecute\n", + " if callable(lActivityItem[\"Def\"]): # CHeck if def is callable\n", + "KeyError: 'Def'\n", + "----------------------------------------\n", + "----------------------------------------\n", + "Exception happened during processing of request from ('127.0.0.1', 57158)\n", + "Traceback (most recent call last):\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\socketserver.py\", line 650, in process_request_thread\n", + " self.finish_request(request, client_address)\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Server.py\", line 476, in finish_request\n", + " HTTPServer.finish_request(self, request, client_address)\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\socketserver.py\", line 360, in finish_request\n", + " self.RequestHandlerClass(request, client_address, self)\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\socketserver.py\", line 720, in __init__\n", + " self.handle()\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\http\\server.py\", line 426, in handle\n", + " self.handle_one_request()\n", + " File \"C:\\Abs\\Archive\\scopeSrcUL\\OpenRPA\\Resources\\WPy64-3720\\python-3.7.2.amd64\\lib\\http\\server.py\", line 414, in handle_one_request\n", + " method()\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Server.py\", line 336, in do_GET\n", + " lAuthenticateDict = AuthenticateVerify(self)\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Server.py\", line 73, in AuthenticateVerify\n", + " \"Password\": lPassword\n", + " File \"..\\pyOpenRPA\\Orchestrator\\ProcessorOld.py\", line 94, in Activity\n", + " lItem[\"Result\"] = Processor.ActivityListExecute(inGSettings = gSettingsDict, inActivityList = [inActivity])\n", + " File \"..\\pyOpenRPA\\Orchestrator\\Processor.py\", line 32, in ActivityListExecute\n", + " if callable(lActivityItem[\"Def\"]): # CHeck if def is callable\n", + "KeyError: 'Def'\n", + "----------------------------------------\n" + ] + } + ], + "source": [ + "from pyOpenRPA.Orchestrator import Orchestrator # Import orchestrator main\n", + "Orchestrator.Orchestrator(inGSettings=gSettings) # Call the orchestrator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.2" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/Sources/Sandbox/OrchestratorTestProcessorNew.ipynb b/Sources/Sandbox/OrchestratorTestProcessorNew.ipynb new file mode 100644 index 00000000..310e29cd --- /dev/null +++ b/Sources/Sandbox/OrchestratorTestProcessorNew.ipynb @@ -0,0 +1,23 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py b/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py index 187feb5d..2320ddc7 100644 --- a/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py +++ b/Sources/pyOpenRPA/Orchestrator/BackwardCompatibility.py @@ -1,6 +1,54 @@ # Def to check inGSettings and update structure to the backward compatibility # !!! ATTENTION: Backward compatibility has been started from v1.1.13 !!! # So you can use config of the orchestrator 1.1.13 in new Orchestrator versions and all will be ok :) (hope it's true) +import win32security + +# v1.2.0 Def for old procesor to new processor +# Return new activity for the new processor +def v1_2_0_ProcessorOld2NewActivityDict(inActivityOld): + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # Technical defs for support old operations in old processor + # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # { + # "Type":"WindowsLogon", + # "Domain":"", + # "User":"", + # "Password":"" + # # Return "Result": True - user is logged on, False - user is not logged on + # } + def WindowLogon(inUserStr, inPasswordStr, inDomainStr=""): + ################################# + # Windows logon + ################################# + try: + hUser = win32security.LogonUser( + inUserStr, + inDomainStr, + inPasswordStr, + win32security.LOGON32_LOGON_NETWORK, + win32security.LOGON32_PROVIDER_DEFAULT + ) + except win32security.error: + return False + else: + return True + ################################### + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # + # # # # # # # # # # # # # # # # # # # # # # # # # # # # + + if inActivityOld["Type"] == "WindowsLogon": + lResult = { + "Def": WindowLogon, # def link or def alias (look gSettings["Processor"]["AliasDefDict"]) + "ArgList":[], # Args list + "ArgDict":{"inUserStr": inActivityOld["User"],"inPasswordStr":inActivityOld["Password"],"inDomainStr":inActivityOld["Domain"]}, # Args dictionary + "ArgGSettings": None # Name of GSettings attribute: str (ArgDict) or index (for ArgList) + } + else: + raise Exception(f"BackwardCompatibility up to 1.2.0, old processor: No type {inActivityOld['Type']} has been found in old processor.") + return lResult # return the result + def Update(inGSettings): lL = inGSettings["Logger"] # Alias for logger # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # diff --git a/Sources/pyOpenRPA/Orchestrator/Orchestrator.py b/Sources/pyOpenRPA/Orchestrator/Orchestrator.py index 7ea775cb..38874e72 100644 --- a/Sources/pyOpenRPA/Orchestrator/Orchestrator.py +++ b/Sources/pyOpenRPA/Orchestrator/Orchestrator.py @@ -41,165 +41,172 @@ def GSettingsAutocleaner(inGSettings): if lL: lL.debug(f"Client > Session > TechnicalSessionGUIDCache > lItemKeyStr: Lifetime is expired. Remove from gSettings") # Info inGSettings["Client"]["Session"]["TechnicalSessionGUIDCache"] = lTechnicalSessionGUIDCacheNew # Set updated Cache # # # # # # # # # # # # # # # # # # # # # # # # # # -#Call Settings function from argv[1] file -################################################ -lSubmoduleFunctionName = "Settings" -lFileFullPath = sys.argv[1] -lModuleName = (lFileFullPath.split("\\")[-1])[0:-3] -lTechSpecification = importlib.util.spec_from_file_location(lModuleName, lFileFullPath) -lTechModuleFromSpec = importlib.util.module_from_spec(lTechSpecification) -lTechSpecificationModuleLoader = lTechSpecification.loader.exec_module(lTechModuleFromSpec) -gSettingsDict = None -if lSubmoduleFunctionName in dir(lTechModuleFromSpec): - # Run SettingUpdate function in submodule - gSettingsDict = getattr(lTechModuleFromSpec, lSubmoduleFunctionName)() -################################################# -#mGlobalDict = Settings.Settings(sys.argv[1]) -#Logger alias -lL = gSettingsDict["Logger"] - -if lL: lL.info("Link the gSettings in submodules") #Logging -Processor.gSettingsDict = gSettingsDict -Timer.gSettingsDict = gSettingsDict -Timer.Processor.gSettingsDict = gSettingsDict -Server.gSettingsDict = gSettingsDict -Server.Processor.gSettingsDict = gSettingsDict - -# Check _SessionLast_RDPList.json in working directory. if exist - load into gsettings -# GSettings -#"RobotRDPActive": { -# "RDPList": { -if os.path.exists("_SessionLast_RDPList.json"): - lFile = open("_SessionLast_RDPList.json", "r", encoding="utf-8") - lSessionLastRDPList = json.loads(lFile.read()) - lFile.close() # Close the file - os.remove("_SessionLast_RDPList.json") # remove the temp file - gSettingsDict["RobotRDPActive"]["RDPList"]=lSessionLastRDPList # Set the last session dict - if lL: lL.warning(f"RDP Session List was restored from previous Orchestrator session") - -#Инициализация настроечных параметров -lDaemonLoopSeconds=gSettingsDict["Scheduler"]["ActivityTimeCheckLoopSeconds"] -lDaemonActivityLogDict={} #Словарь отработанных активностей, ключ - кортеж (, , , ) -lDaemonLastDateTime=datetime.datetime.now() -gSettingsDict["Server"]["WorkingDirectoryPathStr"] = os.getcwd() # Set working directory in g settings - -# Init SettingsUpdate defs from file list (after RDP restore) -lSettingsUpdateFilePathList = gSettingsDict.get("OrchestratorStart", {}).get("DefSettingsUpdatePathList",[]) -lSubmoduleFunctionName = "SettingsUpdate" -lSettingsPath = "\\".join(os.path.join(os.getcwd(), __file__).split("\\")[:-1]) -for lModuleFilePathItem in lSettingsUpdateFilePathList: # Import defs with try catch - try: # Try to init - go next if error and log in logger - lModuleName = lModuleFilePathItem[0:-3] - lFileFullPath = os.path.join(lSettingsPath, lModuleFilePathItem) - lTechSpecification = importlib.util.spec_from_file_location(lModuleName, lFileFullPath) - lTechModuleFromSpec = importlib.util.module_from_spec(lTechSpecification) - lTechSpecificationModuleLoader = lTechSpecification.loader.exec_module(lTechModuleFromSpec) - if lSubmoduleFunctionName in dir(lTechModuleFromSpec): - # Run SettingUpdate function in submodule - getattr(lTechModuleFromSpec, lSubmoduleFunctionName)(gSettingsDict) - except Exception as e: - if lL: lL.exception(f"Error when init .py file in orchestrator '{lModuleFilePathItem}'. Exception is below:") - -# Turn on backward compatibility -BackwardCompatibility.Update(inGSettings= gSettingsDict) - -#Инициализация сервера -lThreadServer = Server.RobotDaemonServer("ServerThread", gSettingsDict) -lThreadServer.start() -if lL: lL.info("Web server has been started") #Logging - -# Init the RobotScreenActive in another thread -lRobotScreenActiveThread = threading.Thread(target= Monitor.CheckScreen) -lRobotScreenActiveThread.daemon = True # Run the thread in daemon mode. -lRobotScreenActiveThread.start() # Start the thread execution. -if lL: lL.info("Robot Screen active has been started") #Logging - -# Init the RobotRDPActive in another thread -lRobotRDPActiveThread = threading.Thread(target= RobotRDPActive.RobotRDPActive, kwargs={"inGSettings":gSettingsDict}) -lRobotRDPActiveThread.daemon = True # Run the thread in daemon mode. -lRobotRDPActiveThread.start() # Start the thread execution. -if lL: lL.info("Robot RDP active has been started") #Logging - -# Init autocleaner in another thread -lAutocleanerThread = threading.Thread(target= GSettingsAutocleaner, kwargs={"inGSettings":gSettingsDict}) -lAutocleanerThread.daemon = True # Run the thread in daemon mode. -lAutocleanerThread.start() # Start the thread execution. -if lL: lL.info("Autocleaner thread has been started") #Logging - -# Orchestrator start activity -if lL: lL.info("Orchestrator start activity run") #Logging -for lActivityItem in gSettingsDict["OrchestratorStart"]["ActivityList"]: - Processor.ActivityListOrDict(lActivityItem) - -if lL: lL.info("Scheduler loop start") #Logging -gDaemonActivityLogDictRefreshSecInt = 10 # The second period for clear lDaemonActivityLogDict from old items -gDaemonActivityLogDictLastTime = time.time() # The second perioad for clean lDaemonActivityLogDict from old items -while True: - lCurrentDateTime = datetime.datetime.now() - #Циклический обход правил - lFlagSearchActivityType=True - # Periodically clear the lDaemonActivityLogDict - if time.time()-gDaemonActivityLogDictLastTime>=gDaemonActivityLogDictRefreshSecInt: - gDaemonActivityLogDictLastTime = time.time() # Update the time - for lIndex, lItem in enumerate(lDaemonActivityLogDict): - if lItem["ActivityEndDateTime"] and lCurrentDateTime<=lItem["ActivityEndDateTime"]: - pass - # Activity is actual - do not delete now + +# Main def for orchestrator +def Orchestrator(inGSettings): + #mGlobalDict = Settings.Settings(sys.argv[1]) + gSettingsDict = inGSettings # Alias for old name in alg + #Logger alias + lL = gSettingsDict["Logger"] + if lL: lL.info("Link the gSettings in submodules") #Logging + Processor.gSettingsDict = gSettingsDict + Timer.gSettingsDict = gSettingsDict + Timer.Processor.gSettingsDict = gSettingsDict + Server.gSettingsDict = gSettingsDict + Server.ProcessorOld.gSettingsDict = gSettingsDict # Backward compatibility + + # Check _SessionLast_RDPList.json in working directory. if exist - load into gsettings + # GSettings + #"RobotRDPActive": { + # "RDPList": { + if os.path.exists("_SessionLast_RDPList.json"): + lFile = open("_SessionLast_RDPList.json", "r", encoding="utf-8") + lSessionLastRDPList = json.loads(lFile.read()) + lFile.close() # Close the file + os.remove("_SessionLast_RDPList.json") # remove the temp file + gSettingsDict["RobotRDPActive"]["RDPList"]=lSessionLastRDPList # Set the last session dict + if lL: lL.warning(f"RDP Session List was restored from previous Orchestrator session") + + #Инициализация настроечных параметров + lDaemonLoopSeconds=gSettingsDict["Scheduler"]["ActivityTimeCheckLoopSeconds"] + lDaemonActivityLogDict={} #Словарь отработанных активностей, ключ - кортеж (, , , ) + lDaemonLastDateTime=datetime.datetime.now() + gSettingsDict["Server"]["WorkingDirectoryPathStr"] = os.getcwd() # Set working directory in g settings + + # Init SettingsUpdate defs from file list (after RDP restore) + lSettingsUpdateFilePathList = gSettingsDict.get("OrchestratorStart", {}).get("DefSettingsUpdatePathList",[]) + lSubmoduleFunctionName = "SettingsUpdate" + lSettingsPath = "\\".join(os.path.join(os.getcwd(), __file__).split("\\")[:-1]) + for lModuleFilePathItem in lSettingsUpdateFilePathList: # Import defs with try catch + try: # Try to init - go next if error and log in logger + lModuleName = lModuleFilePathItem[0:-3] + lFileFullPath = os.path.join(lSettingsPath, lModuleFilePathItem) + lTechSpecification = importlib.util.spec_from_file_location(lModuleName, lFileFullPath) + lTechModuleFromSpec = importlib.util.module_from_spec(lTechSpecification) + lTechSpecificationModuleLoader = lTechSpecification.loader.exec_module(lTechModuleFromSpec) + if lSubmoduleFunctionName in dir(lTechModuleFromSpec): + # Run SettingUpdate function in submodule + getattr(lTechModuleFromSpec, lSubmoduleFunctionName)(gSettingsDict) + except Exception as e: + if lL: lL.exception(f"Error when init .py file in orchestrator '{lModuleFilePathItem}'. Exception is below:") + + # Turn on backward compatibility + BackwardCompatibility.Update(inGSettings= gSettingsDict) + + #Инициализация сервера + lThreadServer = Server.RobotDaemonServer("ServerThread", gSettingsDict) + lThreadServer.start() + if lL: lL.info("Web server has been started") #Logging + + # Init the RobotScreenActive in another thread + lRobotScreenActiveThread = threading.Thread(target= Monitor.CheckScreen) + lRobotScreenActiveThread.daemon = True # Run the thread in daemon mode. + lRobotScreenActiveThread.start() # Start the thread execution. + if lL: lL.info("Robot Screen active has been started") #Logging + + # Init the RobotRDPActive in another thread + lRobotRDPActiveThread = threading.Thread(target= RobotRDPActive.RobotRDPActive, kwargs={"inGSettings":gSettingsDict}) + lRobotRDPActiveThread.daemon = True # Run the thread in daemon mode. + lRobotRDPActiveThread.start() # Start the thread execution. + if lL: lL.info("Robot RDP active has been started") #Logging + + # Init autocleaner in another thread + lAutocleanerThread = threading.Thread(target= GSettingsAutocleaner, kwargs={"inGSettings":gSettingsDict}) + lAutocleanerThread.daemon = True # Run the thread in daemon mode. + lAutocleanerThread.start() # Start the thread execution. + if lL: lL.info("Autocleaner thread has been started") #Logging + + # Orchestrator start activity + if lL: lL.info("Orchestrator start activity run") #Logging + for lActivityItem in gSettingsDict["OrchestratorStart"]["ActivityList"]: + Processor.ActivityListOrDict(lActivityItem) + + if lL: lL.info("Scheduler loop start") #Logging + gDaemonActivityLogDictRefreshSecInt = 10 # The second period for clear lDaemonActivityLogDict from old items + gDaemonActivityLogDictLastTime = time.time() # The second perioad for clean lDaemonActivityLogDict from old items + while True: + lCurrentDateTime = datetime.datetime.now() + #Циклический обход правил + lFlagSearchActivityType=True + # Periodically clear the lDaemonActivityLogDict + if time.time()-gDaemonActivityLogDictLastTime>=gDaemonActivityLogDictRefreshSecInt: + gDaemonActivityLogDictLastTime = time.time() # Update the time + for lIndex, lItem in enumerate(lDaemonActivityLogDict): + if lItem["ActivityEndDateTime"] and lCurrentDateTime<=lItem["ActivityEndDateTime"]: + pass + # Activity is actual - do not delete now + else: + # remove the activity - not actual + lDaemonActivityLogDict.pop(lIndex,None) + lIterationLastDateTime = lDaemonLastDateTime # Get current datetime before iterator (need for iterate all activities in loop) + # Iterate throught the activity list + for lIndex, lItem in enumerate(gSettingsDict["Scheduler"]["ActivityTimeList"]): + # Prepare GUID of the activity + lGUID = None + if "GUID" in lItem and lItem["GUID"]: + lGUID = lItem["GUID"] else: - # remove the activity - not actual - lDaemonActivityLogDict.pop(lIndex,None) - lIterationLastDateTime = lDaemonLastDateTime # Get current datetime before iterator (need for iterate all activities in loop) - # Iterate throught the activity list - for lIndex, lItem in enumerate(gSettingsDict["Scheduler"]["ActivityTimeList"]): - # Prepare GUID of the activity - lGUID = None - if "GUID" in lItem and lItem["GUID"]: - lGUID = lItem["GUID"] - else: - lGUID = str(uuid.uuid4()) - lItem["GUID"]=lGUID - - #Проверка дней недели, в рамках которых можно запускать активность - lItemWeekdayList=lItem.get("WeekdayList", [0, 1, 2, 3, 4, 5, 6]) - if lCurrentDateTime.weekday() in lItemWeekdayList: - if lFlagSearchActivityType: - ####################################################################### - #Branch 1 - if has TimeHH:MM - ####################################################################### - if "TimeHH:MM" in lItem: - #Вид активности - запуск процесса - #Сформировать временной штамп, относительно которого надо будет проверять время - #часовой пояс пока не учитываем - lActivityDateTime=datetime.datetime.strptime(lItem["TimeHH:MM"],"%H:%M") - lActivityDateTime=lActivityDateTime.replace(year=lCurrentDateTime.year,month=lCurrentDateTime.month,day=lCurrentDateTime.day) - #Убедиться в том, что время наступило - if ( - lActivityDateTime>=lDaemonLastDateTime and - lCurrentDateTime>=lActivityDateTime): - # Log info about activity - if lL: lL.info(f"Scheduler:: Activity is started. Scheduler item: {lItem}") #Logging - # Do the activity - Processor.ActivityListOrDict(lItem["Activity"]) - lIterationLastDateTime = datetime.datetime.now() # Set the new datetime for the new processor activity - ####################################################################### - #Branch 2 - if TimeHH:MMStart, TimeHH:MMStop, ActivityIntervalSeconds - ####################################################################### - if "TimeHH:MMStart" in lItem and "TimeHH:MMStop" in lItem and "ActivityIntervalSeconds" in lItem: - #Сформировать временной штамп, относительно которого надо будет проверять время - #часовой пояс пока не учитываем - lActivityDateTime=datetime.datetime.strptime(lItem["TimeHH:MMStart"],"%H:%M") - lActivityDateTime=lActivityDateTime.replace(year=lCurrentDateTime.year,month=lCurrentDateTime.month,day=lCurrentDateTime.day) - lActivityTimeEndDateTime=datetime.datetime.strptime(lItem["TimeHH:MMStop"],"%H:%M") - lActivityTimeEndDateTime=lActivityTimeEndDateTime.replace(year=lCurrentDateTime.year,month=lCurrentDateTime.month,day=lCurrentDateTime.day) - #Убедиться в том, что время наступило - if ( - lCurrentDateTime=lActivityDateTime and - (lGUID,lActivityDateTime) not in lDaemonActivityLogDict): - #Запись в массив отработанных активностей - lDaemonActivityLogDict[(lGUID,lActivityDateTime)]={"ActivityStartDateTime":lCurrentDateTime, "ActivityEndDateTime":lActivityTimeEndDateTime} - #Запуск циклической процедуры - Timer.activityLoopStart(lItem["ActivityIntervalSeconds"], lActivityTimeEndDateTime, lItem["Activity"]) - lDaemonLastDateTime = lIterationLastDateTime # Set the new datetime for the new processor activity - #Уснуть до следующего прогона - time.sleep(lDaemonLoopSeconds) \ No newline at end of file + lGUID = str(uuid.uuid4()) + lItem["GUID"]=lGUID + + #Проверка дней недели, в рамках которых можно запускать активность + lItemWeekdayList=lItem.get("WeekdayList", [0, 1, 2, 3, 4, 5, 6]) + if lCurrentDateTime.weekday() in lItemWeekdayList: + if lFlagSearchActivityType: + ####################################################################### + #Branch 1 - if has TimeHH:MM + ####################################################################### + if "TimeHH:MM" in lItem: + #Вид активности - запуск процесса + #Сформировать временной штамп, относительно которого надо будет проверять время + #часовой пояс пока не учитываем + lActivityDateTime=datetime.datetime.strptime(lItem["TimeHH:MM"],"%H:%M") + lActivityDateTime=lActivityDateTime.replace(year=lCurrentDateTime.year,month=lCurrentDateTime.month,day=lCurrentDateTime.day) + #Убедиться в том, что время наступило + if ( + lActivityDateTime>=lDaemonLastDateTime and + lCurrentDateTime>=lActivityDateTime): + # Log info about activity + if lL: lL.info(f"Scheduler:: Activity is started. Scheduler item: {lItem}") #Logging + # Do the activity + Processor.ActivityListOrDict(lItem["Activity"]) + lIterationLastDateTime = datetime.datetime.now() # Set the new datetime for the new processor activity + ####################################################################### + #Branch 2 - if TimeHH:MMStart, TimeHH:MMStop, ActivityIntervalSeconds + ####################################################################### + if "TimeHH:MMStart" in lItem and "TimeHH:MMStop" in lItem and "ActivityIntervalSeconds" in lItem: + #Сформировать временной штамп, относительно которого надо будет проверять время + #часовой пояс пока не учитываем + lActivityDateTime=datetime.datetime.strptime(lItem["TimeHH:MMStart"],"%H:%M") + lActivityDateTime=lActivityDateTime.replace(year=lCurrentDateTime.year,month=lCurrentDateTime.month,day=lCurrentDateTime.day) + lActivityTimeEndDateTime=datetime.datetime.strptime(lItem["TimeHH:MMStop"],"%H:%M") + lActivityTimeEndDateTime=lActivityTimeEndDateTime.replace(year=lCurrentDateTime.year,month=lCurrentDateTime.month,day=lCurrentDateTime.day) + #Убедиться в том, что время наступило + if ( + lCurrentDateTime=lActivityDateTime and + (lGUID,lActivityDateTime) not in lDaemonActivityLogDict): + #Запись в массив отработанных активностей + lDaemonActivityLogDict[(lGUID,lActivityDateTime)]={"ActivityStartDateTime":lCurrentDateTime, "ActivityEndDateTime":lActivityTimeEndDateTime} + #Запуск циклической процедуры + Timer.activityLoopStart(lItem["ActivityIntervalSeconds"], lActivityTimeEndDateTime, lItem["Activity"]) + lDaemonLastDateTime = lIterationLastDateTime # Set the new datetime for the new processor activity + #Уснуть до следующего прогона + time.sleep(lDaemonLoopSeconds) + +# Backward compatibility below to 1.2.0 +if __name__ == "__main__": + #Call Settings function from argv[1] file + ################################################ + lSubmoduleFunctionName = "Settings" + lFileFullPath = sys.argv[1] + lModuleName = (lFileFullPath.split("\\")[-1])[0:-3] + lTechSpecification = importlib.util.spec_from_file_location(lModuleName, lFileFullPath) + lTechModuleFromSpec = importlib.util.module_from_spec(lTechSpecification) + lTechSpecificationModuleLoader = lTechSpecification.loader.exec_module(lTechModuleFromSpec) + gSettingsDict = None + if lSubmoduleFunctionName in dir(lTechModuleFromSpec): + # Run SettingUpdate function in submodule + gSettingsDict = getattr(lTechModuleFromSpec, lSubmoduleFunctionName)() + ################################################# + Orchestrator(inGSettings=gSettingsDict) # Call the orchestrator diff --git a/Sources/pyOpenRPA/Orchestrator/Processor.py b/Sources/pyOpenRPA/Orchestrator/Processor.py index e14eec4d..c060db0c 100644 --- a/Sources/pyOpenRPA/Orchestrator/Processor.py +++ b/Sources/pyOpenRPA/Orchestrator/Processor.py @@ -8,8 +8,8 @@ def ProcessorRunSync(inGSettings): # { # "Def":"DefAliasTest", # def link or def alias (look gSettings["Processor"]["AliasDefDict"]) # "ArgList":[1,2,3], # Args list - # "ArgDict":{"ttt":1,"222":2,"dsd":3} # Args dictionary - # "ArgGSettings": # Name of GSettings attribute: str (ArgDict) or index (for ArgList) + # "ArgDict":{"ttt":1,"222":2,"dsd":3}, # Args dictionary + # "ArgGSettings": None # Name of GSettings attribute: str (ArgDict) or index (for ArgList) # }, ], "AliasDefDict": {}, # Storage for def with Str alias. To use it see pyOpenRPA.Orchestrator.ControlPanel diff --git a/Sources/pyOpenRPA/Orchestrator/ProcessorOld.py b/Sources/pyOpenRPA/Orchestrator/ProcessorOld.py index 2e64edb8..3bf8da19 100644 --- a/Sources/pyOpenRPA/Orchestrator/ProcessorOld.py +++ b/Sources/pyOpenRPA/Orchestrator/ProcessorOld.py @@ -1,12 +1,14 @@ import datetime import http.client import json -import pdb import os import sys import subprocess import importlib import psutil +from . import BackwardCompatibility # Use backward compatibility to goes to the new processor +from . import Processor # Use new processor +import copy #Input arg # [ # { @@ -84,8 +86,14 @@ import psutil # "Result" gSettingsDict = None def Activity(inActivity): - #Глобальная переменная - глобальный словарь унаследованный от Settings.py + lItem = copy.deepcopy(inActivity) global gSettingsDict + # Update 2020.10 - go to the new processor + lActivityNew = BackwardCompatibility.v1_2_0_ProcessorOld2NewActivityDict(inActivityOld=inActivity) + # Append new activity in list + lItem["Result"] = Processor.ActivityListExecute(inGSettings = gSettingsDict, inActivityList = [inActivity]) + """ + #Глобальная переменная - глобальный словарь унаследованный от Settings.py lL = gSettingsDict["Logger"] # Alias for logger #Alias (compatibility) lItem = inActivity @@ -271,6 +279,7 @@ def Activity(inActivity): else: lItem["Result"] = True ################################### + """ #Вернуть результат return lItem diff --git a/Sources/pyOpenRPA/Orchestrator/Server.py b/Sources/pyOpenRPA/Orchestrator/Server.py index 43080607..db1b3406 100644 --- a/Sources/pyOpenRPA/Orchestrator/Server.py +++ b/Sources/pyOpenRPA/Orchestrator/Server.py @@ -11,7 +11,8 @@ from socketserver import ThreadingMixIn import threading import json from threading import Thread -from . import Processor +from . import Processor # Add new processor +from . import ProcessorOld # Support old processor - deprecated defs only for backward compatibility import urllib.parse # decode URL in string import importlib import pdb @@ -64,7 +65,7 @@ def AuthenticateVerify(inRequest): lDomain = lUser.split("\\")[0] lUser = lUser.split("\\")[1] #Try to logon - use processor - lLogonResult = Processor.Activity( + lLogonResult = ProcessorOld.Activity( { "Type": "WindowsLogon", "Domain": lDomain, @@ -451,7 +452,7 @@ class testHTTPServer_RequestHandler(BaseHTTPRequestHandler): if not lIsSuperToken: if lL: lL.info(f"Server:: User activity from web. Domain: {self.OpenRPA['Domain']}, Username: {self.OpenRPA['User']}, Activity: {lInputObject}") # Send message back to client - message = json.dumps(Processor.ActivityListOrDict(lInputObject)) + message = json.dumps(ProcessorOld.ActivityListOrDict(lInputObject)) # Write content as utf-8 data self.wfile.write(bytes(message, "utf8")) return diff --git a/changelog.md b/changelog.md index 6adbdf74..f8210cd1 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,7 @@ [1.2.0] - Consolidated processor from old orchestrator and from RDPActive processor (one threaded). Look in GSettings - - Support old orchestrator structure Processor. +- - - Create BackwardCompatibility def to update structure from old to new processor - - Support orchestrator start - - Support scheduler - - Support old items