Merge branch 'master' of https://github.com/SmartDataAnalytics/OpenResearch

Author: WolfgangFahl

migration/tests/corpusfortesting.py

@@ -3,21 +3,20 @@
 
 @author: wf
 '''
-from datasources.openresearch import OREventCorpus, OREventManager
+from corpus.lookup import CorpusLookup
 from lodstorage.storageconfig import StorageConfig
-
 from ormigrate.toolbox import HelperFunctions as hf
 from wikifile.wikiFileManager import WikiFileManager
 from os import path
 import os
+
+
 class CorpusForTesting(object):
     '''
     Simplify initializing an EventCorpus for tests (singleton)
     '''
-
     wikiId='orclone'
 
-    
     @classmethod
     def hasCache(cls):
         '''
@@ -48,29 +47,70 @@ def getWikiFileManager(cls,wikiId=None,debug=False):
         return wikiFileManager
 
     @classmethod
-    def getEventCorpusFromWikiAPI(cls, wikiId=None, force=False, debug=False):
+    def getEventDataSourceFromWikiAPI(cls, lookupId:str= "orclone", forceUpdate:bool=False, debug:bool=False):
         '''
         get events with series by knitting / linking the entities together
+
+        Args:
+            lookupId(str): ID of the EventDataSource that should be returned.
+            forceUpdate(bool): True if the data should be fetched from the source instead of the cache
+            debug(bool): If True display debug output
+
+        Returns:
+            EventDataSource
         '''
-        wikiUser=cls.getWikiUser(wikiId)
-        config = cls.getStorageConfig()
-        eventCorpus=OREventCorpus(config,debug=debug)
-        eventCorpus.fromCache(wikiUser,force=force)
-        eventCorpus.wikiFileManager=cls.getWikiFileManager(wikiId, debug)
-        return eventCorpus
+        eventDataSource=cls.getEventDataSource(lookupId=lookupId, forceUpdate=forceUpdate, debug=debug)
+        return eventDataSource
 
     @classmethod
-    def getEventCorpusFromWikiText(cls,wikiId=None,debug=False):
+    def getEventDataSourceFromWikiText(cls, lookupId:str= "orclone-backup", forceUpdate:bool=False, debug=False):
         """
         get events with series by knitting/linking entities from a WikiFileManager
+
+        Args:
+            lookupId(str): ID of the EventDataSource that should be returned.
+            forceUpdate(bool): True if the data should be fetched from the source instead of the cache
+            debug(bool): If True display debug output
+
+        Returns:
+            EventDataSource
         """
-        if wikiId is None:
-            wikiId=cls.wikiId
-        config=cls.getStorageConfig()
-        wikiFileManager=cls.getWikiFileManager(wikiId,debug)
-        eventCorpus=OREventCorpus(config,debug=debug)
-        eventCorpus.fromWikiFileManager(wikiFileManager)
-        return eventCorpus
+        eventDataSource = cls.getEventDataSource(lookupId=lookupId, forceUpdate=forceUpdate, debug=debug)
+        return eventDataSource
+
+    @classmethod
+    def getEventDataSource(cls, lookupId:str, forceUpdate:bool=False, debug:bool=False):
+        """
+
+        Args:
+            lookupId(str): ID of the EventDataSource that should be returned.
+            forceUpdate(bool): True if the data should be fetched from the source instead of the cache
+            debug(bool): If True display debug output
+
+        Returns:
+            EventDataSource
+        """
+        lookup = CorpusLookup(lookupIds=[lookupId], configure=cls.patchEventSource, debug=debug)
+        lookup.load(forceUpdate=forceUpdate)
+        eventDataSource = lookup.getDataSource(lookupId)
+        return eventDataSource
+
+    @classmethod
+    def patchEventSource(cls, lookup:CorpusLookup):
+        '''
+        patches the EventManager and EventSeriesManager by adding wikiUser and WikiFileManager
+        '''
+        wikiUser = cls.getWikiUser(cls.wikiId)
+        wikiFileManager = cls.getWikiFileManager(cls.wikiId)
+        for lookupId in ["orclone", "orclone-backup", "or", "or-backup"]:
+            orDataSource = lookup.getDataSource(lookupId)
+            if orDataSource is not None:
+                if lookupId.endswith("-backup"):
+                    orDataSource.eventManager.wikiFileManager = wikiFileManager
+                    orDataSource.eventSeriesManager.wikiFileManager = wikiFileManager
+                else:
+                    orDataSource.eventManager.wikiUser = wikiUser
+                    orDataSource.eventSeriesManager.wikiUser = wikiUser
 
     @classmethod
     def getStorageConfig(cls):

migration/tests/testIssue168_ratingCallBack.py

@@ -66,7 +66,7 @@ def testPageRatingList(self):
     #    test the rating call back
     #    '''
     #    profile=Profiler("test rating call back")
-    #    eventCorpus=Corpus.getEventCorpusFromWikiAPI(debug=self.debug)
+    #    eventCorpus=Corpus.getEventDataSourceFromWikiAPI(debug=self.debug)
     #    lod,errors=eventCorpus.eventList.getRatedLod(Event.rateMigration)
     #    self.checkRatedLod(lod, errors)
     #    lod,errors=eventCorpus.eventSeriesList.getRatedLod(EventSeries.rateMigration)

migration/tests/testIssue170_Curation.py

@@ -8,7 +8,7 @@
 from ormigrate.issue170_curation import CurationQualityChecker
 from tests.corpusfortesting import CorpusForTesting as Corpus
 from collections import Counter
-from tests.pagefixtoolbox import PageFixerToolbox, PageFixerTest
+from tests.pagefixtoolbox import PageFixerTest
 
 class TestIssue170(PageFixerTest):
     '''
@@ -47,8 +47,8 @@ def testCurationQualityCheck170(self):
 
     def testUserCount(self):
         # only needed to setup userrating yaml file
-        eventCorpus=Corpus.getEventCorpusFromWikiAPI(debug=self.debug, force=True)
-        userLookup=eventCorpus.eventManager.getLookup("lastEditor",withDuplicates=True)
+        eventDataSource=Corpus.getEventDataSourceFromWikiAPI(debug=self.debug, forceUpdate=True)
+        userLookup=eventDataSource.eventManager.getLookup("lastEditor",withDuplicates=True)
         if self.debug:
             print (f"{len(userLookup)} users")
         expected=1 if hf.inPublicCI() else 140

migration/tests/testIssue236_CSV.py

@@ -52,8 +52,8 @@ class TestEventCorpus(unittest.TestCase):
     def setUp(self):
         self.debug = False
         self.profile=True
-        self.eventCorpusAPI=None
-        self.eventCorpusWikiText=None
+        self.eventDataSourceAPI=None
+        self.eventDataSourceWikiText=None
         pass
 
     def tearDown(self):
@@ -85,31 +85,31 @@ def testEventCorpusFromWikiUser(self):
         '''
         self.debug=True
         profile=Profiler("getting EventCorpus from WikiUser")
-        eventCorpus=Corpus.getEventCorpusFromWikiAPI(debug=self.debug, force=True)
+        eventDataSource=Corpus.getEventDataSourceFromWikiAPI(forceUpdate=True, debug=self.debug)
         profile.time()
-        self.checkEventCorpus(eventCorpus)
-        
-        
+        self.checkEventCorpus(eventDataSource)
+
+
     def testEventCorpusFromWikiUserCache(self):
         """
         test the Event Corpus from the wikiUser(API) cache.
         """
         debug = True
         if Corpus.hasCache():
             profile=Profiler(f"getting EventCorpus for {Corpus.wikiId} from WikiUser Cache",self.profile)
-            self.eventCorpusAPI=Corpus.getEventCorpusFromWikiAPI(debug=debug, force=False)
+            self.eventDataSourceAPI=Corpus.getEventDataSourceFromWikiAPI(debug=debug, forceUpdate=False)
             profile.time()
-            self.checkEventCorpus(self.eventCorpusAPI)
+            self.checkEventCorpus(self.eventDataSourceAPI)
 
 
     def testEventCorpusFromWikiFileManager(self):
         """
         test the Event Corpus from the wiki file manager(wikiFiles).
         """
         profile=Profiler(f"getting EventCorpus from wikiText files for {Corpus.wikiId}")
-        self.eventCorpusWikiText = Corpus.getEventCorpusFromWikiText(debug=self.debug)
+        self.eventDataSourceWikiText = Corpus.getEventDataSourceFromWikiText(forceUpdate=True, debug=self.debug)
         profile.time()
-        self.checkEventCorpus(self.eventCorpusWikiText,['pageTitle'])
+        self.checkEventCorpus(self.eventDataSourceWikiText, ['pageTitle'])
 
 
     def testMatchingSetsForEventCorpus(self):
@@ -119,15 +119,15 @@ def testMatchingSetsForEventCorpus(self):
         if not Corpus.hasCache():
             return
         profile=Profiler(f"getting EventCorpora from wikiAPI and wikiText files for {Corpus.wikiId}")
-        if self.eventCorpusAPI is None:
-            self.eventCorpusAPI=Corpus.getEventCorpusFromWikiAPI(debug=self.debug, force=False)
-        if self.eventCorpusWikiText is None:
-            self.eventCorpusWikiText = Corpus.getEventCorpusFromWikiText(debug=self.debug)
+        if self.eventDataSourceAPI is None:
+            self.eventDataSourceAPI=Corpus.getEventDataSourceFromWikiAPI(debug=self.debug, forceUpdate=False)
+        if self.eventDataSourceWikiText is None:
+            self.eventDataSourceWikiText = Corpus.getEventDataSourceFromWikiText(debug=self.debug)
         profile.time()    
         profile=Profiler(f"finding common events and series for {Corpus.wikiId}")
         keys=["acronym","pageTitle"]
-        eventSet=MatchingSet("Events","api",self.eventCorpusAPI.eventManager,"wikiText",self.eventCorpusWikiText.eventManager,keys)
-        eventSeriesSet=MatchingSet("EventSeries","api",self.eventCorpusAPI.eventSeriesManager,"wikiText",self.eventCorpusWikiText.eventSeriesManager,keys)
+        eventSet=MatchingSet("Events","api", self.eventDataSourceAPI.eventManager, "wikiText", self.eventDataSourceWikiText.eventManager, keys)
+        eventSeriesSet=MatchingSet("EventSeries","api", self.eventDataSourceAPI.eventSeriesManager, "wikiText", self.eventDataSourceWikiText.eventSeriesManager, keys)
         profile.time()
         eventSet.showStats()
         eventSeriesSet.showStats()

migration/tests/test_EventCorpus.py

@@ -17,7 +17,7 @@ def setUp(self) -> None:
         targetWikiTextPath = f"{home}/.or/generated/Location"
         self.wikiFileManager = CorpusForTesting.getWikiFileManager()
         self.wikiFileManager.targetPath=targetWikiTextPath
-        self.eventCorpus=CorpusForTesting.getEventCorpusFromWikiText()
+        self.eventDataSource=CorpusForTesting.getEventDataSourceFromWikiText()
         self.eventLocationContext=EventLocationHandler(wikiFileManager=self.wikiFileManager)
 
     def test_generateLocationPages(self):
@@ -37,7 +37,7 @@ def test_generateORLocationPages(self):
         Note: Not running in CI since it generates a lot of pages and uses functionalities which are test in the other tests
         """
         profile = Profiler("Generate OPENRESEARCH location pages (Limited to 100 events)", self.profile)
-        self.eventLocationContext.generateORLocationPages(self.eventCorpus.eventManager.events[:100], overwrite=True)
+        self.eventLocationContext.generateORLocationPages(self.eventDataSource.eventManager.events[:100], overwrite=True)
         profile.time()
         # ToDo: test if generated correctly