Merge branch 'master' of https://github.com/SmartDataAnalytics/OpenResearch

Author: WolfgangFahl

migration/ormigrate/EventLocationContext.py

@@ -52,7 +52,8 @@ def generateORLocationPages(self, events:list, overwrite:bool=False, limit:int=N
         # Generate country location pages
         self.generateLocationPages(self.locationContext.countries, overwrite)
         # Fix locations of the event → Normalized location names afterwards
-        self.locationFixer.fixEvents(events)
+        for event in events:
+            self.locationFixer.fixEventRecord(event.__dict__)
 
         cityCounter=EventLocationContext.getFieldCounter(events, 'city', 'City')
         usedCities=[]

migration/ormigrate/issue166_cfp.py

@@ -12,6 +12,7 @@
 from wikifile.wikiFile import WikiFile
 from smw.pagefixer import PageFixerManager
 from ormigrate.fixer import ORFixer
+from smw.rating import EntityRating
 
 
 class WikiCFPIDFixer(ORFixer):
@@ -73,46 +74,34 @@ def getWikiCFPIdFromPage(self, eventWikiText):
             return wikicfpid
         return None
 
-    def fixEventFileFromWiki(self,pageTitle):
+    def fix(self, rating:EntityRating):
         """
-        Get the pageTitle from the wiki directly and run the fixer on it
+        Get the entityRating object and apply the fixer on it.
         
         Args:
-            pageTitle(str): page title of a wiki page
-        
-        Returns:
-            wikiFile(WikiFile): A WikiFile object if fixer is applied, None otherwise
+            entityRating(EntityRating): EntityRating object of the Event
         """
-        wikiFileManager = self.wikiFileManager
-        wikiFile = wikiFileManager.getWikiFile(pageTitle)
-        event = str(wikiFile.wikiText)
-        wikicfpid= self.getWikiCFPIdFromPage(event)
+        wikiFile = rating.wikiFile
+        eventRecord = rating.getRecord()
+        wikiText = str(wikiFile.wikiText)
+        wikicfpid= self.getWikiCFPIdFromPage(wikiText)
         if wikicfpid is not None:
-            values = {}
-            values['wikicfpId'] = wikicfpid
-            wikiFile.add_template('Event',values)
-            return wikiFile
-        return None
+            eventRecord['wikicfpId'] = wikicfpid
 
-    def fixEventFile(self,path,event):
+    def rate(self, rating: EntityRating):
         """
-            Get the path and content of .wiki file and run the fixer on it
-            Args:
-                path(str): path of .wiki file
-                event(str): content of .wiki file
-            Returns:
-                wikiFile(WikiFile): A WikiFile object if fixer is applied, None otherwise
+        Rate the rating object as per the issue 166
+        Args:
+            rating(EntityRating): EntityRating object of the Event
         """
-        filename = ntpath.basename(path).replace('.wiki','')
-        wikiFilePath = ntpath.dirname(path)
-        wikicfpid= self.getWikiCFPIdFromPage(event)
-        if wikicfpid is not None:
-            wikiFile = WikiFile(filename,wikiFilePath)
-            values = {}
-            values['wikicfpId']=wikicfpid
-            wikiFile.add_template('Event',values)
-            return wikiFile
-        return None
+        wikiFile= rating.wikiFile
+        eventWikiText = str(wikiFile.wikiText)
+        wikiCFPId = self.getWikiCFPIdFromPage(eventWikiText)
+        if wikiCFPId is None:
+            rating.set(1, RatingType.ok, "no legacy wikiCFP import id  found")
+        else:
+            rating.set(5, RatingType.invalid, f"legacy wikiCFP reference {wikiCFPId} found")
+        return rating
 
 
     def fixPageWithDBCrosscheck(self, wikiText, wikicfpid):
@@ -143,7 +132,9 @@ def fixPageWithDBCrosscheck(self, wikiText, wikicfpid):
         #                print('Title or acronym not found')
         #                print(dic)
         #                print(orEvent)
-    
+
+
+    #TODO Change function when architecture is implemented.
     def getRatingFromWikiFile(self,wikiFile:WikiFile)->PageRating:
         '''
         Args:

migration/ormigrate/issue220_location.py

@@ -1,8 +1,7 @@
 from geograpy.locator import LocationContext, Location, City, Country, Region
-from openresearch.event import Event
 from smw.pagefixer import PageFixerManager
 from ormigrate.fixer import ORFixer
-from smw.rating import Rating, RatingType
+from smw.rating import Rating, RatingType, EntityRating
 
 
 class LocationFixer(ORFixer):
@@ -23,41 +22,21 @@ def __init__(self,pageFixerManager):
         '''
         super(LocationFixer, self).__init__(pageFixerManager)
         LocationFixer.locationContext=self.getORLocationContext()
- 
-    def fixEventRecords(self, events:list):
-        """
-        Gets list of dicts (list of events) and tries to fix the location entries
-        """
-        count=0
-        stats={}
-        for event_unfixed in events:
-            event, errors = self.fixEventRecord(event_unfixed)
-            print(errors)
-            if errors is not None:
-                for error in errors.keys():
-                    if error in stats:
-                        stats[error]+=1
-                    else:
-                        stats[error]=1
-        print(stats)
 
-    def fixEvents(self, events:list):
+    def fix(self,rating:EntityRating):
         '''
-        fixes the location of the given events
-
-        Args:
-            events(list): list of Event objects that should be fixed
+        tries fixing the location entries of the  given entity
         '''
-        for event in events:
-            self.fixEvent(event)
+        eventRecord = rating.getRecord()
+        self.fixEventRecord(eventRecord)
 
-    def fixEvent(self, event:Event):
+    def fixEventRecord(self, event:dict, errors:dict=None, bestFit=True):
         '''
-        fixes the location of the given event
+        Args:
+            event(dict): event records containing the location values that should be fixed
+            errors(dict): dictonary containing the errors of the given event record → new errors are added to the dict
+            bestFit(bool): If true the best/closed fit for a location is chosen (e.g. city with highest population). Otherwise a fix is only applied if the location can be identified with certainty.
         '''
-        self.fixEventRecord(event.__dict__)
-
-    def fixEventRecord(self, event:dict, errors=None):
         # event location values
         if errors is None:
             errors = {}
@@ -115,21 +94,22 @@ def fixEventRecord(self, event:dict, errors=None):
                     cities = list(filter(
                         lambda x: 'wikidataid' in x.country.__dict__ and x.country.wikidataid in countryids,
                         cities))
-                if len(cities) == 1:
+                if len(cities) >0:
                     # city can be identified (Could still be incorrect)
-                    final_city=cities[0]
+                    if len(cities) > 1:
+                        errors["city_unclear"] = f"City '{event_city}' matches against multiple cities in the LocationCorpus. Other location information are not sufficient enough to clearly identify the city"
+                        cities=sorted(cities,key=lambda city:0 if 'population'not in city.__dict__ or city.population is None else int(city.population), reverse=True)
+                    final_city = cities[0]
                     if isinstance(final_city,City):
                         event[self.CITY]=self.getPageTitle(final_city)
                         event[self.REGION] = self.getPageTitle(final_city.region)
                         event[self.COUNTRY] = self.getPageTitle(final_city.country)
                         errors["complete"]="Location of event complete"
                         return event, errors
-                elif len(cities) == 0:
+                else:
                     # No matching city -> Two possibilities: event location information incorrect or locations missing in LocationCorpus
                     errors["city_unknown"]=f"City '{event_city}' could not be matched against a city in the LocationCorpus with the given region and country. Either location information are incorrect or location is missing in the corpus."
                     #return None, errors
-                else:
-                    errors["city_unclear"]=f"City '{event_city}' matches against multiple cities in the LocationCorpus. Other location information are not sufficient enough to clearly identify the city"
             else:
                 # event_city is not the LocationCorpus
                 isPossiblyMisplaced=True
@@ -314,11 +294,17 @@ def getPageTitle(location:Location):
             pageTitle=location.name
         return pageTitle
 
-    @classmethod
-    def getRating(cls, eventRecord):
+
+    def rate(self, rating:EntityRating):
         '''
         get the pain Rating for the given eventRecord
         '''
+        eventRecord=rating.getRecord()
+        arating=self.getRating(eventRecord)
+        rating.set(arating.pain, arating.reason, arating.hint)
+
+    @classmethod
+    def getRating(cls, eventRecord):
         painrating = None
         city = None
         region = None
@@ -329,24 +315,24 @@ def getRating(cls, eventRecord):
         if cls.COUNTRY in eventRecord: country = eventRecord[cls.COUNTRY]
         if not city and not region and not country:
             # location is not defined
-            painrating = Rating(7, RatingType.missing,f'Locations are not defined')
+            painrating=Rating(7, RatingType.missing,f'Locations are not defined')
         else:
             if 'locationContext' in cls.__dict__:
                 cities=cls.__dict__['locationContext'].getCities(city)
                 regions=cls.__dict__['locationContext'].getCities(region)
                 countries=cls.__dict__['locationContext'].getCities(country)
             if cities and regions and countries:
                 # all locations are recognized
-                painrating= Rating(1,RatingType.ok,f'Locations are valid. (Country: {country}, Region: {region}, City:{city})')
+                painrating=Rating(1,RatingType.ok,f'Locations are valid. (Country: {country}, Region: {region}, City:{city})')
             elif not cities:
                 # City is not valid
-                painrating = Rating(6, RatingType.invalid,f'City is not recognized. (City:{city})')
+                painrating=Rating(6, RatingType.invalid,f'City is not recognized. (City:{city})')
             elif not regions:
                 # City is valid but region is not
-                painrating = Rating(5, RatingType.invalid,f'Region is not recognized. (Country: {country}, Region: {region}, City:{city})')
+                painrating=Rating(5, RatingType.invalid,f'Region is not recognized. (Country: {country}, Region: {region}, City:{city})')
             else:
                 # City and region are valid but country is not
-                painrating = Rating(3, RatingType.invalid,f'Country is not recognized. (Country: {country}, Region: {region}, City:{city})')
+                painrating=Rating(3, RatingType.invalid,f'Country is not recognized. (Country: {country}, Region: {region}, City:{city})')
         return painrating
 
     @staticmethod

migration/tests/testIssue166_WikiCfpID.py

@@ -7,6 +7,9 @@
 from ormigrate.issue166_cfp import WikiCFPIDFixer
 from openresearch.event import Event
 from tests.pagefixtoolbox import PageFixerTest
+from wikifile.wikiFileManager import WikiFileManager
+from wikifile.wikiFile import WikiFile
+from smw.rating import EntityRating
 
 class TestWikiCFPId(PageFixerTest):
     '''
@@ -20,32 +23,52 @@ def setUp(self):
 
     def testIssue166Examples(self):
         """
-        Tests the issue 166 for addition of WikiCFP-ID to applicable pages
+        Tests the issue 166 for addition of WikiCFP-ID to applicable pages.
+        Testing the fix function
+
         """
         fixer=self.getPageFixer()
         samplesWikiText = Event.getSampleWikiTextList()
         wikicfpid= fixer.getWikiCFPIdFromPage(samplesWikiText[1])
         self.assertIsNotNone(wikicfpid)
         self.assertEqual(wikicfpid,'3845')
 
-        
-        samplesDict=Event.getSamples()
-        count=0
-        for sample in samplesDict:
-            wikiFile= fixer.fixEventFileFromWiki(sample['pageTitle'])
-            if wikiFile is not None:
-                count+=1
-        self.assertGreaterEqual(count,1)
+        wikiFile = WikiFile('sampleFile',None,samplesWikiText)
+        event = Event()
+        event.wikiFile = wikiFile
+        entityRating = EntityRating(event)
+        entityRating.pageTitle='Test'
+        entityRating.templateName= 'Event'
+
+        # Get Fixer
+        fixer = self.getPageFixer()
+
+        # Rate With Fixer
+        fixer.rate(entityRating)
+        if self.debug:
+            print(entityRating)
+        self.assertEqual(entityRating.pain, 5)
+
+        # Fix With Fixer
+        fixer.fix(entityRating)
+        self.assertEqual(entityRating.entity.wikicfpId,"3845")
 
         # TODO
-        if fixer.databaseAvailable():
-            fixedPage= fixer.fixPageWithDBCrosscheck('test', samplesWikiText[1], wikicfpid)
-            if self.debug:
-                print(fixedPage)
-            fixedDict=fixedPage.extract_template('Event')
-            self.assertIsNotNone(fixedDict['wikicfpId'])
-            self.assertEqual(fixedDict['wikicfpId'],'3845')
-        
+        # if fixer.databaseAvailable():
+        #     fixedPage= fixer.fixPageWithDBCrosscheck(samplesWikiText[1], wikicfpid)
+        #     if self.debug:
+        #         print(fixedPage)
+        #     fixedDict=fixedPage.extract_template('Event')
+        #     self.assertIsNotNone(fixedDict['wikicfpId'])
+        #     self.assertEqual(fixedDict['wikicfpId'],'3845')
+
+    def testIssue166Rating(self):
+        """
+        test the Rating function of the fixer
+        """
+
+
+    # TODO Change function when architecture is implemented.
     def testIssue166(self):
         '''
         test the wikicfpID handling