Skip to content

Commit 92192d2

Browse files
laurenbarkerchrisseto
laurenbarker
authored and
chrisseto
committed
[SHARE-739][Improvement] Check quality of OAI sources (#660)
* Add comment in config if DSpace or Digital Commons Update base urls Add tests for sources Add source stat model Save source stat information Update sources who have corrected their earliestDatestamp Catch and log all source exceptions Add admin page for source stats Use one model for source stats Don't save the model instance twice Refactor task logic Add grade field * Update sources that have fixed their earliest datestamp * Resolve migration conflicts
1 parent c72afdc commit 92192d2

123 files changed

Lines changed: 567 additions & 146 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎project/settings.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,12 @@
329329
'RawData Janitor': {
330330
'task': 'share.janitor.tasks.rawdata_janitor',
331331
'schedule': crontab(minute=0) # hourly
332-
}
332+
},
333+
'Source Stats': {
334+
'task': 'share.tasks.source_stats',
335+
'schedule': crontab(minute=0, hour='3,9,15,21'), # every 6 hours
336+
'args': (),
337+
},
333338
}
334339

335340
CELERY_RESULT_EXPIRES = 60 * 60 * 24 * 3 # 4 days

‎share/admin/__init__.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from share.models.logs import HarvestLog
2929
from share.models.meta import Subject, SubjectTaxonomy
3030
from share.models.registration import ProviderRegistration
31+
from share.models.sources import SourceStat
3132

3233

3334
admin.site.register(AbstractCreativeWork, CreativeWorkAdmin)
@@ -347,6 +348,36 @@ def recursive_link_list(subjects):
347348
subject_links.short_description = 'Subjects'
348349

349350

351+
class SourceStatAdmin(admin.ModelAdmin):
352+
search_fields = ('config__label', 'config__source__long_title')
353+
list_display = ('label', 'date_created', 'base_urls_match', 'earliest_datestamps_match', 'response_elapsed_time', 'response_status_code', 'grade_')
354+
list_filter = ('grade', 'response_status_code', 'config__label')
355+
356+
GRADE_COLORS = {
357+
0: 'red',
358+
5: 'orange',
359+
10: 'green',
360+
}
361+
GRADE_LETTERS = {
362+
0: 'F',
363+
5: 'C',
364+
10: 'A',
365+
}
366+
367+
def source(self, obj):
368+
return obj.config.source.long_title
369+
370+
def label(self, obj):
371+
return obj.config.label
372+
373+
def grade_(self, obj):
374+
return format_html(
375+
'<span style="font-weight: bold; color: {}">{}</span>',
376+
self.GRADE_COLORS[obj.grade],
377+
self.GRADE_LETTERS[obj.grade],
378+
)
379+
380+
350381
admin.site.unregister(AccessToken)
351382
admin.site.register(AccessToken, AccessTokenAdmin)
352383

@@ -362,4 +393,5 @@ def recursive_link_list(subjects):
362393
admin.site.register(Source, SourceAdmin)
363394
admin.site.register(SourceConfig, SourceConfigAdmin)
364395
admin.site.register(SubjectTaxonomy, SubjectTaxonomyAdmin)
396+
admin.site.register(SourceStat, SourceStatAdmin)
365397
admin.site.register(Transformer)

‎share/migrations/0039_sourcestat.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.11.1 on 2017-06-13 17:46
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations, models
6+
import django.db.models.deletion
7+
8+
9+
class Migration(migrations.Migration):
10+
11+
dependencies = [
12+
('share', '0038_auto_20170606_1857'),
13+
]
14+
15+
operations = [
16+
migrations.CreateModel(
17+
name='SourceStat',
18+
fields=[
19+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
20+
('is_deleted', models.BooleanField(default=False)),
21+
('date_created', models.DateTimeField(auto_now_add=True)),
22+
('response_status_code', models.SmallIntegerField(blank=True, null=True)),
23+
('response_elapsed_time', models.FloatField(blank=True, null=True)),
24+
('response_exception', models.TextField(blank=True, null=True)),
25+
('earliest_datestamp_config', models.DateField(blank=True, null=True)),
26+
('base_url_config', models.TextField()),
27+
('admin_note', models.TextField(blank=True)),
28+
('grade', models.FloatField()),
29+
('earliest_datestamp_source', models.DateField(blank=True, null=True)),
30+
('earliest_datestamps_match', models.BooleanField(default=False)),
31+
('base_url_source', models.TextField(blank=True, null=True)),
32+
('base_urls_match', models.BooleanField(default=False)),
33+
('config', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.SourceConfig')),
34+
],
35+
),
36+
]

‎share/migrations/0043_merge_20170626_1516.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# -*- coding: utf-8 -*-
2+
# Generated by Django 1.11.1 on 2017-06-26 15:16
3+
from __future__ import unicode_literals
4+
5+
from django.db import migrations
6+
7+
8+
class Migration(migrations.Migration):
9+
10+
dependencies = [
11+
('share', '0042_merge_20170620_1330'),
12+
('share', '0039_sourcestat'),
13+
]
14+
15+
operations = [
16+
]

‎share/models/__init__.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
from share.models.registration import * # noqa
1010
from share.models.identifiers import * # noqa
1111
from share.models.relations import * # noqa
12-
from share.models.banner import * # noqa
13-
from share.models.ingest import * # noqa
14-
from share.models.logs import * # noqa
12+
from share.models.banner import * # noqa
13+
from share.models.ingest import * # noqa
14+
from share.models.logs import * # noqa
15+
from share.models.sources import * # noqa
1516
from share.models.celery import * # noqa

‎share/models/sources.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import logging
2+
3+
from django.db import models
4+
5+
from share.models.ingest import SourceConfig
6+
7+
logger = logging.getLogger(__name__)
8+
__all__ = ('SourceStat',)
9+
10+
11+
class SourceStat(models.Model):
12+
config = models.ForeignKey(SourceConfig, on_delete=models.CASCADE)
13+
is_deleted = models.BooleanField(default=False)
14+
date_created = models.DateTimeField(auto_now_add=True)
15+
response_status_code = models.SmallIntegerField(blank=True, null=True)
16+
response_elapsed_time = models.FloatField(blank=True, null=True)
17+
response_exception = models.TextField(blank=True, null=True)
18+
earliest_datestamp_config = models.DateField(blank=True, null=True)
19+
base_url_config = models.TextField()
20+
admin_note = models.TextField(blank=True)
21+
grade = models.FloatField()
22+
23+
# OAI specific
24+
earliest_datestamp_source = models.DateField(blank=True, null=True)
25+
earliest_datestamps_match = models.BooleanField(default=False)
26+
27+
base_url_source = models.TextField(blank=True, null=True)
28+
base_urls_match = models.BooleanField(default=False)
29+
30+
def __repr__(self):
31+
return '<{}({}, {})>'.format(self.__class__.__name__, self.pk, self.config.label)
32+
33+
def __str__(self):
34+
return '{}: {}'.format(self.config.source.long_title, self.config.label)

‎share/sources/au.uow/source.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1+
# Digital Commons/Bepress
12
configs:
23
- base_url: http://ro.uow.edu.au/do/oai/
34
disabled: false
4-
earliest_date: null
5+
earliest_date: 2000-01-19T00:00:00Z
56
harvester: oai
67
harvester_kwargs: {metadata_prefix: oai_dc}
78
label: au.uow

‎share/sources/be.ghent/source.yaml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,27 @@
11
configs:
22
- base_url: https://biblio.ugent.be/oai
3-
disabled: true
4-
earliest_date: null
3+
disabled: false
4+
earliest_date: 2016-12-14T15:38:10Z
55
harvester: oai
6-
harvester_kwargs: {metadata_prefix: oai_dc}
7-
label: be.ghent
6+
harvester_kwargs: {metadata_prefix: mods}
7+
label: be.ghent.mods
88
rate_limit_allowance: 1
99
rate_limit_period: 2
10-
transformer: oai_dc
10+
transformer: mods
1111
transformer_kwargs:
1212
approved_sets: null
1313
emitted_type: CreativeWork
1414
property_list: []
1515
type_map: {}
1616
- base_url: https://biblio.ugent.be/oai
17-
disabled: false
17+
disabled: true
1818
earliest_date: null
1919
harvester: oai
20-
harvester_kwargs: {metadata_prefix: mods}
21-
label: be.ghent.mods
20+
harvester_kwargs: {metadata_prefix: oai_dc}
21+
label: be.ghent
2222
rate_limit_allowance: 1
2323
rate_limit_period: 2
24-
transformer: mods
24+
transformer: oai_dc
2525
transformer_kwargs:
2626
approved_sets: null
2727
emitted_type: CreativeWork

‎share/sources/br.pcurio/source.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
configs:
2-
- base_url: http://www.maxwell.vrac.puc-rio.br/DC_Todos.php
2+
- base_url: https://www.maxwell.vrac.puc-rio.br/DC_Todos.php
33
disabled: false
4-
earliest_date: null
4+
earliest_date: null # earliestDatestamp is earliest published
55
harvester: oai
66
harvester_kwargs: {metadata_prefix: oai_dc, time_granularity: false}
77
label: br.pcurio

‎share/sources/ca.umontreal/source.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1+
# DSpace
12
configs:
23
- base_url: http://papyrus.bib.umontreal.ca/oai/request
34
disabled: false
4-
earliest_date: null
5+
earliest_date: 2005-05-18T18:27:23Z
56
harvester: oai
67
harvester_kwargs: {metadata_prefix: mods}
78
label: ca.umontreal.mods

0 commit comments

Comments
 (0)