From 5d0b3ecb50700cce6a25068f1e69d17b34c75910 Mon Sep 17 00:00:00 2001 From: Capocchi L Date: Sun, 10 May 2020 17:42:18 +0200 Subject: [PATCH 1/2] improve the relaod model function --- ZipManager.py | 201 +++++++++++++++----------------------------------- 1 file changed, 58 insertions(+), 143 deletions(-) diff --git a/ZipManager.py b/ZipManager.py index cffc6c74..4e42d35e 100644 --- a/ZipManager.py +++ b/ZipManager.py @@ -32,11 +32,11 @@ #global Cmtp #Cmtp=0 -def walk_reload(module: types.ModuleType) -> None: - if hasattr(module, "__all__"): - for submodule_name in module.__all__: - walk_reload(getattr(module, submodule_name)) - importlib.reload(module) +def get_from_modules(name:str)->types.ModuleType: + for s,m in sys.modules.items(): + if name in s: + return m + return None def module_list(topdir): ret = [] @@ -52,87 +52,6 @@ def module_list(topdir): ret.append('.'.join([modpath,os.path.splitext(f)[0]])) return ret -def relative_imports(pyfile): - import_pattern = re.compile(r'^\s*from\s+\.([\w\.]*)\s+import\s*' + - r'(?:\(([\w, \t\n\r\f\v]+)\)|([\w, \t\r\f\v]+))', re.M) - as_pattern = re.compile(r'(\w+)\s+as\s+\w+') - ret = {} - with open(pyfile,'rt') as fin: - for m in import_pattern.finditer(fin.read()): - src, tgt = m.group(1), m.group(2) or m.group(3) - tgts = [] - for t in tgt.split(','): - t = t.strip() - m = as_pattern.match(t) - tgts.append(t if m is None else m.group(1)) - if src not in ret: - ret[src] = [] - ret[src] += tgts - return ret - -def imported_modules(modpath, pyfile): - ret = [] - for k,v in relative_imports(pyfile).items(): - if k == '': - ret += ['.'.join([modpath,i]) for i in v] - else: - s = re.search(r'[^.]',k) - if s is None: - relmod = '.'.join(modpath.split('.')[:-len(k)]) - ret += ['.'.join([relmod,i]) for i in v] - else: - n = s.start() - if n == 0: - relmod = modpath - else: - relmod = '.'.join(modpath.split('.')[:-n]) - ret.append(relmod+'.'+k[n:]) - return ret - -def reloadall(fn): - fn_dir = os.path.dirname(fn) + os.sep - module_visit = {fn} - - def reload_recursive_ex(fn): - if zipfile.is_zipfile(fn): - Zip.ClearCache(fn) - - module_name = getPythonModelFileName(fn) - importer = zipimport.zipimporter(fn) - - p = os.path.dirname(os.path.dirname(fn)) - - if p not in sys.path: - sys.path.append(p) - - fullname = "".join([os.path.basename(os.path.dirname(fn)), module_name.split('.py')[0]]) - module = importer.load_module(module_name.split('.py')[0]) - module.__name__ = path_to_module(module_name) - - #if fullname in sys.modules: - # del sys.modules[fullname] - - sys.modules[fullname] = module - else: - import Components - module = Components.BlockFactory.GetModule(fn) - #importlib.reload(module) - import ReloadModule - ReloadModule.recompile(module.__name__) - - - for module_child in vars(module).values(): - if isinstance(module_child, types.ModuleType): - fn_child = getattr(module_child, "__file__", None) - if (fn_child is not None) and fn_child.startswith(fn_dir): - if fn_child not in module_visit: - print("reloading:", fn_child, "from", module) - module_visit.add(fn_child) - - reload_recursive_ex(fn_child) - - return reload_recursive_ex(fn) - def getPythonModelFileName(fn): """ Get filename of zipped python file """ @@ -182,6 +101,15 @@ def __init__(self, fn, files = []): ### local copy self.fn = fn + ### get module name + try: + self.module_name = getPythonModelFileName(fn) + except Exception as info: + sys.stderr.write(_("Error in ZipManager class for GetModule: no python file in the archive\n")) + + ### + self.fullname = "".join([os.path.basename(os.path.dirname(self.fn)), self.module_name.split('.py')[0]]) + if files != []: self.Create(files) @@ -360,7 +288,7 @@ def HasPlugin(fn): def HasTests(fn): """ TODO: comment """ - name = os.path.basename(getPythonModelFileName(fn)).split('.')[0] + name = os.path.basename(self.module_name.split('.'))[0] zf = zipfile.ZipFile(fn, 'r') nl = zf.namelist() zf.close() @@ -383,43 +311,41 @@ def GetTests(fn): # ------------------------------------------------------------------------------ def GetModule(self, rcp=False): - """ Load module from zip file corresponding to the amd or cmd model. + """ Return module from zip file corresponding to the amd or cmd model. It used when the tree library is created. + If the module refered by self.fn is already imported, its returned else its imported using zipimport """ - # get module name - try: - module_name = getPythonModelFileName(self.fn) - except Exception as info: - sys.stderr.write(_("Error in ZipManager class for GetModule: no python file in the archive\n")) - return info - # if necessary, recompile (for update after editing code source of model) #if rcp: recompile(module_name) trigger_event("IMPORT_STRATEGIES", fn=self.fn) - - fullname = "".join([os.path.basename(os.path.dirname(self.fn)), module_name.split('.py')[0]]) - if fullname not in sys.modules: - - p = os.path.dirname(os.path.dirname(self.fn)) - if p not in sys.path: - sys.path.append(p) - - importer = zipimport.zipimporter(self.fn) - module = importer.load_module(module_name.split('.py')[0]) - module.__name__ = path_to_module(module_name) - - ### allows to import with a reference from the parent directory (like parentName.model). - ### Now import of .amd or .cmd module is composed by DomainModel (no point!). - ### Example : import CollectorMessageCollector - sys.modules[fullname] = module - - return module + if self.fullname not in sys.modules: + return self.LoadModule() else: - return sys.modules[fullname] + return sys.modules[self.fullname] + def LoadModule(self): + """ Loead module from zip file corresponding to the amd or cmd model. + """ + ### allows to import the lib from its name (like import MyModel.amd). Dangerous because confuse! + ### Import can be done using: import Name (ex. import MessageCollector - if MessageCollecor is .amd or .cmd) + p = os.path.dirname(os.path.dirname(self.fn)) + if p not in sys.path: + sys.path.append(p) + + importer = zipimport.zipimporter(self.fn) + module = importer.load_module(self.module_name.split('.py')[0]) + module.__name__ = path_to_module(self.module_name) + + ### allows to import with a reference from the parent directory (like parentName.model). + ### Now import of .amd or .cmd module is composed by DomainModel (no point!). + ### Example : import CollectorMessageCollector + sys.modules[self.fullname] = module + + return module + def Recompile(self): """ recompile module from zip file """ @@ -428,37 +354,26 @@ def Recompile(self): # import module try: - module_name = getPythonModelFileName(self.fn) - fullname = "".join([os.path.basename(os.path.dirname(self.fn)), module_name.split('.py')[0]]) - - # for i in [ a for a in module_list(DOMAIN_PATH) if os.path.basename(os.path.dirname(self.fn)) in a]: - # if i in sys.modules: - # impoprlib.reload(sys.modules[i]) - - #reloadall(self.fn) + ### realod submodule from module dependencies! + module = sys.modules[self.fullname] + domain_name = os.path.basename(os.path.dirname(self.fn)) + for name in dir(module): + if type(getattr(module, name)) == types.ModuleType: + ### TODO: only reload the local package (not 'sys' and so one) + importlib.reload(get_from_modules(name)) + + ### reload submodule from directory + #for i in [ a for a in module_list(DOMAIN_PATH) if os.path.basename(os.path.dirname(self.fn)) in a]: + # a = ".".join(i.split('.')[1:]) + # if a in sys.modules: + # importlib.reload(sys.modules[a]) ### clear to clean the import after exporting model (amd or cmd) and reload within the same instance of DEVSimPy - #zipimport._zip_directory_cache.clear() - - ### allows to import the lib from its name (like import MyModel.amd). Dangerous because confuse! - ### Import can be done using: import Name (ex. import MessageCollector - if MessageCollecor is .amd or .cmd) -# p = os.path.dirname(os.path.dirname(self.fn)) -# if p not in sys.path: -# sys.path.append(p) - print('dd') -# importer = zipimport.zipimporter(self.fn) -# module = importer.load_module(module_name.split('.py')[0]) -# module.__name__ = path_to_module(module_name) - - ### allows to import with a reference from the parent directory (like parentName.model). - ### Now import of .amd or .cmd module is composed by DomainModel (no point!). - ### Example : import CollectorMessageCollector -# sys.modules[fullname] = module - - #walk_reload(sys.modules[fullname]) - ### TODO make a recursive method to go up until the Domain dir, for not external lib! - - return sys.modules[fullname] + zipimport._zip_directory_cache.clear() + + ### reload module + module = self.LoadModule() + except Exception as info: msg_i = _("Error in execution: ") msg_o = listf(format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) From 064ff56bf454c93ad6e1b08bf62c99d302b19e9b Mon Sep 17 00:00:00 2001 From: Capocchi L Date: Mon, 11 May 2020 15:22:06 +0200 Subject: [PATCH 2/2] implementation of the update of libraries from LibraryTree --- LibraryTree.py | 63 +++++++++++++++++++++++++++++++++++++++++-------- Menu.py | 5 ++++ ReloadModule.py | 3 +-- Utilities.py | 19 ++++++++++++--- ZipManager.py | 54 +++++++++++++----------------------------- 5 files changed, 92 insertions(+), 52 deletions(-) diff --git a/LibraryTree.py b/LibraryTree.py index ef52e1c0..4e7a21a0 100644 --- a/LibraryTree.py +++ b/LibraryTree.py @@ -29,11 +29,11 @@ import inspect import zipfile import subprocess - +import importlib import Container import Menu -from Utilities import replaceAll, getPYFileListFromInit, path_to_module, printOnStatusBar, NotificationMessage, install_and_import +from Utilities import replaceAll, getPYFileListFromInit, path_to_module, printOnStatusBar, NotificationMessage, install_and_import, module_list from Decorators import BuzyCursorNotification from Components import BlockFactory, DEVSComponent, GetClass from ZipManager import Zip, getPythonModelFileName @@ -311,6 +311,39 @@ def OnDelete(self, evt): else: wx.MessageBox(_("No library selected!"),_("Delete Manager")) + def UpdateSubLib(self, path:str)->bool: + """ Do update lib. + """ + ### reload .py module from path + for s in module_list(path): + module_name = ".".join(s.split('.')[1:]) + if module_name in sys.modules: + module = sys.modules[module_name] + dirname = os.path.dirname(module.__file__) + try: + ### .amd or .cmd + if zipfile.is_zipfile(dirname): + zf = Zip(dirname) + if isinstance(zf.ReImport(), Exception): + return False + else: + importlib.reload(module) + except: + return False + return True + + ### + @BuzyCursorNotification + def OnUpdateSubLib(self, evt): + """ ReImport all module (.py and .amd/.cmd) in lib. + """ + item = self.GetFocusedItem() + path = self.GetItemPyData(item) + if self.UpdateSubLib(path): + NotificationMessage(_('Information'), _('%s has been updated!')%os.path.basename(path), parent=self, timeout=5) + else: + NotificationMessage(_('Error'), _('%s has not been updated! See traceback for more information')%os.path.basename(path), parent=self, timeout=5) + ### def OnNewModel(self, evt): """ New model action has been invoked. @@ -922,28 +955,38 @@ def UpdateDomain(self, path): def OnUpdateAll(self, event): """ Update all imported domain """ - self.UpdateAll() - - NotificationMessage(_('Information'), _("All librairies have been succeffully updated!"), self, timeout=5) + result = self.UpdateAll() + if len(result) == 0: + NotificationMessage(_('Information'), _("All libraries have been succeffully updated!"), self, timeout=5) + else: + NotificationMessage(_('Error'), _("The following libraries updates crash:\n %s")%" \n".join(map(os.path.basename,result)), self, timeout=5) def OnMCCClick(self, event): """ """ tb = event.GetEventObject() LibraryTree.COMPARE_BY_MACABE_METRIC = not tb.GetToolState(Menu.ID_MCC_LIB) - self.UpdateAll() - + self.OnUpdateAll(event) + ### def UpdateAll(self): - """ Update all loaded libaries. + """ Update all loaded libraries. """ + fault = set() ### update all Domain for item in self.GetItemChildren(self.GetRootItem()): - self.UpdateDomain(self.GetPyData(item)) + path = self.GetItemPyData(item) + if self.UpdateSubLib(path): + self.UpdateDomain(self.GetPyData(item)) + else: + fault.add(path) ### to sort domain wx.CallAfter(self.SortChildren,self.root) + + return fault + #self.SortChildren(self.root) ### @@ -1108,4 +1151,4 @@ def OnItemDocumentation(self, evt): def OnInfo(self, event): """ """ - wx.MessageBox(_('Libraries Import Manager.\nYou can import, refresh or upgrade librairies using right options.\nDefault libraries directory is %s.')%(DOMAIN_PATH)) + wx.MessageBox(_('Libraries Import Manager.\nYou can import, refresh or upgrade libraries using right options.\nDefault libraries directory is %s.')%(DOMAIN_PATH)) diff --git a/Menu.py b/Menu.py index 6a3b9e47..7dbccff5 100644 --- a/Menu.py +++ b/Menu.py @@ -151,6 +151,7 @@ ID_UPDATE_LIB = wx.NewIdRef() ID_HELP_LIB = wx.NewIdRef() ID_NEW_MODEL_LIB = wx.NewIdRef() +ID_UPDATE_SUBLIB = wx.NewIdRef() ID_DELETE_LIB = wx.NewIdRef() # Attribute popup menu identifiers @@ -721,7 +722,11 @@ def __init__(self, parent): new_model = wx.MenuItem(self, ID_NEW_MODEL_LIB, _('New Model'), _('Add a new model to the selected library')) new_model.SetBitmap(wx.Bitmap(os.path.join(ICON_PATH_16_16, 'new.png'))) InsertItem(0, new_model) + update_lib = wx.MenuItem(self, ID_UPDATE_SUBLIB, _('Update'), _('Update all models of the selected library')) + update_lib.SetBitmap(wx.Bitmap(os.path.join(ICON_PATH_16_16, 'db_refresh2.png'))) + InsertItem(1, update_lib) self.Bind(wx.EVT_MENU, parent.OnNewModel, id=ID_NEW_MODEL_LIB) + self.Bind(wx.EVT_MENU, parent.OnUpdateSubLib, id=ID_UPDATE_SUBLIB) else: diff --git a/ReloadModule.py b/ReloadModule.py index 78a701ff..9ab7377d 100644 --- a/ReloadModule.py +++ b/ReloadModule.py @@ -25,9 +25,8 @@ def recompile(modulename): if os.path.isfile(modulename) and os.path.exists(modulename): import ZipManager zf = ZipManager.Zip(modulename) - return zf.Recompile() + return zf.ReImport() else: - try: ### first, see if the module can be imported at all... name, ext = os.path.splitext(modulename) diff --git a/Utilities.py b/Utilities.py index 0596c873..7e304245 100644 --- a/Utilities.py +++ b/Utilities.py @@ -36,6 +36,7 @@ import imp import tempfile import pathlib + #import shlex from copy import deepcopy from datetime import datetime @@ -127,7 +128,7 @@ def NotificationMessage(title,message,parent,flag=wx.ICON_INFORMATION, timeout=F else: notify.Show() -def now(): +def now()->str: """ Returns the current time formatted. """ t = time.localtime(time.time()) @@ -135,8 +136,20 @@ def now(): return st - -def shortNow(): +def module_list(topdir:str)->[str]: + for root,dirs,files in os.walk(topdir): + modpath = os.path.basename(topdir) + r = os.path.relpath(root,topdir) + if r != '.': + modpath += '.' + r + for extension in ('*.py', '*.amd', '*.cmd'): + for f in fnmatch.filter(files, extension): + if f == '__init__.py': + yield modpath + elif f not in ['__main__.py']: + yield '.'.join([modpath,os.path.splitext(f)[0]]) + +def shortNow()->str: """ Returns the current time formatted. """ t = time.localtime(time.time()) diff --git a/ZipManager.py b/ZipManager.py index 4e42d35e..e271926c 100644 --- a/ZipManager.py +++ b/ZipManager.py @@ -20,7 +20,6 @@ import inspect import types import importlib -import fnmatch import gettext _ = gettext.gettext @@ -33,27 +32,15 @@ #Cmtp=0 def get_from_modules(name:str)->types.ModuleType: + """ get module with the correct name from the name that come from dir(). + """ for s,m in sys.modules.items(): if name in s: return m return None -def module_list(topdir): - ret = [] - for root,dirs,files in os.walk(topdir): - modpath = os.path.basename(topdir) - r = os.path.relpath(root,topdir) - if r != '.': - modpath += '.' + r - for f in fnmatch.filter(files, '*.py'): - if f == '__init__.py': - ret.append(modpath) - elif f not in ['__main__.py']: - ret.append('.'.join([modpath,os.path.splitext(f)[0]])) - return ret - -def getPythonModelFileName(fn): - """ Get filename of zipped python file +def getPythonModelFileName(fn:str)->str: + """ Get filename of zipped python file. """ #global Cmtp @@ -260,7 +247,7 @@ def GetImage(self, scaleW=16, scaleH=16): return None @staticmethod - def GetPluginFile(fn): + def GetPluginFile(fn:str)->str: """ TODO: comment """ ### zipfile (amd or cmd) @@ -272,7 +259,7 @@ def GetPluginFile(fn): return L.pop(0)[0] if L != [] else "" @staticmethod - def HasPlugin(fn): + def HasPlugin(fn:str)->bool: """ TODO: comment """ @@ -280,18 +267,21 @@ def HasPlugin(fn): zf = zipfile.ZipFile(fn, 'r') nl = zf.namelist() zf.close() + ### plugin file is plugins.pi in root of zipfile or in plugins zipedd directory return any([re.search("^(plugins[/]*[\w]*.py)$", s) for s in nl]) # BDD Test---------------------------------------------------------------------- @staticmethod - def HasTests(fn): + def HasTests(fn:str)->bool: """ TODO: comment """ - name = os.path.basename(self.module_name.split('.'))[0] + module_name = getPythonModelFileName(fn) + name = os.path.basename(module_name.split('.'))[0] zf = zipfile.ZipFile(fn, 'r') nl = zf.namelist() zf.close() + return any([re.search("^(BDD/[\w*/]*\.py|BDD/[\w*/]*\.feature)$", s) for s in nl]) @staticmethod @@ -300,7 +290,6 @@ def GetTests(fn): """ zf = zipfile.ZipFile(fn, 'r') nl = zf.namelist() - zf.close() ### @@ -321,13 +310,10 @@ def GetModule(self, rcp=False): trigger_event("IMPORT_STRATEGIES", fn=self.fn) - if self.fullname not in sys.modules: - return self.LoadModule() - else: - return sys.modules[self.fullname] + return self.ImportModule() if self.fullname not in sys.modules else sys.modules[self.fullname] - def LoadModule(self): - """ Loead module from zip file corresponding to the amd or cmd model. + def ImportModule(self): + """ Import module from zip file corresponding to the amd or cmd model. """ ### allows to import the lib from its name (like import MyModel.amd). Dangerous because confuse! ### Import can be done using: import Name (ex. import MessageCollector - if MessageCollecor is .amd or .cmd) @@ -346,8 +332,8 @@ def LoadModule(self): return module - def Recompile(self): - """ recompile module from zip file + def ReImport(self): + """ Reimport the module from zip file. """ Zip.ClearCache(self.fn) @@ -361,18 +347,12 @@ def Recompile(self): if type(getattr(module, name)) == types.ModuleType: ### TODO: only reload the local package (not 'sys' and so one) importlib.reload(get_from_modules(name)) - - ### reload submodule from directory - #for i in [ a for a in module_list(DOMAIN_PATH) if os.path.basename(os.path.dirname(self.fn)) in a]: - # a = ".".join(i.split('.')[1:]) - # if a in sys.modules: - # importlib.reload(sys.modules[a]) ### clear to clean the import after exporting model (amd or cmd) and reload within the same instance of DEVSimPy zipimport._zip_directory_cache.clear() ### reload module - module = self.LoadModule() + module = self.ImportModule() except Exception as info: msg_i = _("Error in execution: ")