PPHA-713: Script to create email export list for incentive (#443)

# What is the change?

Created a script to run weekly to:
* Import partner data - new records only
* Export email addresses, given name and family name for participants to
receive incentives.
* Create a new questions_incentivised table to timestamp when a
participant email address has been exported.


# Why are we making this change?
* To create a list of email addresses that are sent to incentive
partner. They will use the email address to send the incentive email.

# Diagram of tables and their relationships for this query
<img width="1099" height="1154" alt="Lung_IncentiveExportTable"
src="https://github.com/user-attachments/assets/4a32ecfb-6446-4f27-b621-fbde478c711e"
/>

Author: stephhou

lung_cancer_screening/questions/migrations/0012_alter_whenyouquitsmokingresponse_value_incentivised.py

@@ -0,0 +1,39 @@
+# Generated by Django 6.0.3 on 2026-04-13 15:15
+
+import django.core.validators
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('questions', '0011_alter_whenyouquitsmokingresponse_response_set'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='whenyouquitsmokingresponse',
+            name='value',
+            field=models.IntegerField(validators=[django.core.validators.MinValueValidator(1)]),
+        ),
+        migrations.CreateModel(
+            name='Incentivised',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('incentivised_at', models.DateTimeField(auto_now_add=True)),
+                ('response_set', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='incentivised_record', to='questions.responseset')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='incentivised_records', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='incentivised',
+            constraint=models.UniqueConstraint(fields=('user',), name='questions_incentivised_unique_user'),
+        ),
+    ]

lung_cancer_screening/questions/migrations/0013_remove_incentivised_questions_incentivised_unique_user_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 6.0.4 on 2026-04-24 12:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('questions', '0012_alter_whenyouquitsmokingresponse_value_incentivised'),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name='incentivised',
+            name='questions_incentivised_unique_user',
+        ),
+        migrations.AddConstraint(
+            model_name='incentivised',
+            constraint=models.UniqueConstraint(fields=('user',), name='unique_incentive_per_user'),
+        ),
+    ]

lung_cancer_screening/questions/models/__init__.py

@@ -12,6 +12,7 @@
 from .gender_response import GenderResponse # noqa: F401
 from .have_you_ever_smoked_response import HaveYouEverSmokedResponse # noqa: F401
 from .height_response import HeightResponse # noqa: F401
+from .incentivised import Incentivised # noqa: F401
 from .periods_when_you_stopped_smoking_response import PeriodsWhenYouStoppedSmokingResponse # noqa: F401
 from .relatives_age_when_diagnosed_response import RelativesAgeWhenDiagnosedResponse # noqa: F401
 from .respiratory_conditions_response import RespiratoryConditionsResponse # noqa: F401

lung_cancer_screening/questions/models/incentivised.py

@@ -0,0 +1,20 @@
+
+from django.db import models
+
+from .base import BaseModel
+from .response_set import ResponseSet
+
+
+class Incentivised(BaseModel):
+    user = models.ForeignKey('questions.User', on_delete=models.CASCADE, related_name='incentivised_records')
+    response_set = models.OneToOneField(ResponseSet, on_delete=models.CASCADE, related_name='incentivised_record')
+
+    incentivised_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["user"],
+                name="unique_incentive_per_user",
+            )
+        ]

scripts/sql/export_email_addresses_for_incentives.sql

@@ -0,0 +1,77 @@
+-- Commands are written on one line for psql execution, can be formatted for readability when running in a SQL client.
+
+-- This script performs the following steps:
+-- 1 - imports new data from csv file. The csv file is expected to be a full export of all partner records, but only new records will be inserted into the permanent table.
+-- 2 - creates a list of email addresses of participants eligible for incentives
+-- 3 - updates participants that have been exported so that they are timestamped as having received an incentive
+
+-- Incentive eligibility is based on:
+-- - having submitted the questionnaire
+-- - not having already received an incentive
+-- - Online submitted_at date is earlier than telephone questionnaire date_conducted.
+
+-- Partner import strategy:
+-- The partner sends a full CSV export every week. Rather than replacing all data each run, we insert only
+-- new records using a UNIQUE index on (nhs_number, conducted_at) and ON CONFLICT DO NOTHING.
+-- Records removed from the partner's source will NOT be deleted.
+
+
+-- Steps to follow:
+-- 1. Log into AVD.
+-- 2. Upload csv file to AVD
+-- 3. Find file in RemoteVirtualDrive and copy to accessible location for psql COPY command.
+-- 4. PATH_TO_FILE - search for this and replace with actual file path.
+-- 5. Login to DB in AVD.
+
+
+-- ============================================================
+-- RUN ONCE: Create permanent partner import table
+-- Only run this block on first setup. The unique constraint on
+-- (nhs_number, conducted_at) prevents duplicate rows being
+-- inserted on subsequent weekly loads.
+-- ============================================================
+CREATE TABLE IF NOT EXISTS inhealth_partner_data (nhs_number TEXT, date_of_birth TEXT, date_conducted TEXT, conducted_at TIMESTAMPTZ, smoking_status TEXT, average_cigarettes_per_day_while_smoking TEXT, duration_smoked_years TEXT, years_since_quitting_smoking TEXT, height_measurement_type TEXT, height_measurement_value_metric_cm TEXT, weight_measurement_type TEXT, weight_measurement_value_metric_kg TEXT, previous_respiratory_diagnosis TEXT, personal_history_of_previous_cancer TEXT, family_history_of_lung_cancer TEXT, personal_history_of_asthma TEXT, asbestos_exposure_from_job_or_activity TEXT, education TEXT, ethnicity TEXT, plco_lung_cancer_risk_score TEXT, llp_lung_cancer_risk_score TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), CONSTRAINT uq_partner_nhs_conducted_at UNIQUE (nhs_number, conducted_at));
+
+
+-- ============================================================
+-- RUN WEEKLY: Load new partner records from CSV
+-- Step 1: Import CSV into a temporary staging table.
+-- Step 2: Insert only rows where (nhs_number, conducted_at)
+--         do not already exist in the permanent table.
+--         Existing rows are silently skipped (DO NOTHING).
+-- ============================================================
+
+-- Step 1: Create staging table and load CSV
+CREATE TEMP TABLE tmp_incentive_partner_staging (nhs_number TEXT, date_of_birth TEXT, date_conducted TEXT, smoking_status TEXT, average_cigarettes_per_day_while_smoking TEXT, duration_smoked_years TEXT, years_since_quitting_smoking TEXT, height_measurement_type TEXT, height_measurement_value_metric_cm TEXT, weight_measurement_type TEXT, weight_measurement_value_metric_kg TEXT, previous_respiratory_diagnosis TEXT, personal_history_of_previous_cancer TEXT, family_history_of_lung_cancer TEXT, personal_history_of_asthma TEXT, asbestos_exposure_from_job_or_activity TEXT, education TEXT, ethnicity TEXT, plco_lung_cancer_risk_score TEXT, llp_lung_cancer_risk_score TEXT);
+
+-- Copy data from file into staging table - update PATH_TO_FILE before running
+
+\copy tmp_incentive_partner_staging (nhs_number, date_of_birth, date_conducted, smoking_status, average_cigarettes_per_day_while_smoking, duration_smoked_years, years_since_quitting_smoking, height_measurement_type, height_measurement_value_metric_cm, weight_measurement_type, weight_measurement_value_metric_kg, previous_respiratory_diagnosis, personal_history_of_previous_cancer, family_history_of_lung_cancer, personal_history_of_asthma, asbestos_exposure_from_job_or_activity, education, ethnicity, plco_lung_cancer_risk_score, llp_lung_cancer_risk_score) FROM 'PATH_TO_FILE' WITH (FORMAT csv, HEADER true);
+
+-- Step 2: Insert all rows from staging. Existing rows with the same (nhs_number, conducted_at)
+--         are skipped.
+INSERT INTO inhealth_partner_data (nhs_number, date_of_birth, date_conducted, conducted_at, smoking_status, average_cigarettes_per_day_while_smoking, duration_smoked_years, years_since_quitting_smoking, height_measurement_type, height_measurement_value_metric_cm, weight_measurement_type, weight_measurement_value_metric_kg, previous_respiratory_diagnosis, personal_history_of_previous_cancer, family_history_of_lung_cancer, personal_history_of_asthma, asbestos_exposure_from_job_or_activity, education, ethnicity, plco_lung_cancer_risk_score, llp_lung_cancer_risk_score) SELECT nhs_number, date_of_birth, date_conducted, to_timestamp(NULLIF(date_conducted, ''), 'DD/MM/YYYY HH24:MI')::timestamptz, smoking_status, average_cigarettes_per_day_while_smoking, duration_smoked_years, years_since_quitting_smoking, height_measurement_type, height_measurement_value_metric_cm, weight_measurement_type, weight_measurement_value_metric_kg, previous_respiratory_diagnosis, personal_history_of_previous_cancer, family_history_of_lung_cancer, personal_history_of_asthma, asbestos_exposure_from_job_or_activity, education, ethnicity, plco_lung_cancer_risk_score, llp_lung_cancer_risk_score FROM tmp_incentive_partner_staging ON CONFLICT (nhs_number, conducted_at) DO NOTHING;
+
+-- Delete temporary staging table
+DROP TABLE IF EXISTS tmp_incentive_partner_staging;
+
+
+-- TRANSACTION START for exporting eligible participants for incentives and updating incentivised table.
+-- Update PATH_TO_EXPORT_FILE before running.
+
+\r
+BEGIN;
+CREATE TEMP TABLE tmp_eligible_incentive_export AS WITH canonical_users AS (SELECT DISTINCT ON (nhs_number) id, email, given_name, family_name, nhs_number FROM questions_user WHERE nhs_number IS NOT NULL ORDER BY nhs_number ASC, created_at DESC) SELECT DISTINCT ON (cu.nhs_number) cu.id AS user_id, qrs.id AS response_set_id, cu.email, cu.given_name, cu.family_name FROM canonical_users cu JOIN questions_responseset qrs ON qrs.user_id = cu.id JOIN inhealth_partner_data ipd ON ipd.nhs_number = cu.nhs_number WHERE ipd.conducted_at > qrs.submitted_at::timestamptz AND NOT EXISTS (SELECT 1 FROM questions_incentivised qi JOIN questions_user iu ON iu.id = qi.user_id WHERE iu.nhs_number = cu.nhs_number) ORDER BY cu.nhs_number ASC, qrs.submitted_at DESC, qrs.id DESC;
+\copy (SELECT email, given_name, family_name  FROM tmp_eligible_incentive_export ORDER BY family_name) TO 'PATH_TO_EXPORT_FILE' WITH (FORMAT csv, HEADER true);
+INSERT INTO questions_incentivised (created_at, updated_at, incentivised_at, user_id, response_set_id) SELECT now(), now(), now(), user_id, response_set_id FROM tmp_eligible_incentive_export;
+SELECT count(*) AS rows_exported_and_marked FROM tmp_eligible_incentive_export;
+
+-- If happy with the Select result type COMMIT; if not, ROLLBACK; to undo changes;
+
+-- COMMIT;
+-- ROLLBACK;
+
+-- TRANSACTION END
+
+-- DELETE temporary export table
+DROP TABLE IF EXISTS tmp_eligible_incentive_export;