Extract succesor school info from GIAS data

GIAS provides a separate "links" file that contains all the successor and predecessor information for schools.
This is helpful when we deiscover a school is closed and has migrated to a different URN. Eventually we want to handle these cases automatcially perhaps adding the successor school to the team and having the info ready gets us one step closer to that.

Author: murugapl

app/lib/mavis_cli/gias/check_import.rb

@@ -24,14 +24,18 @@ def call(input_file:, **)
                 .merge(Session.scheduled)
                 .pluck(:urn)
             ),
-          closed: Set.new,
-          closing: Set.new,
+          closed: {
+          },
+          closing: {
+          },
           year_group_changes: {
           }
         }
         schools_without_future_sessions = {
-          closed: Set.new,
-          closing: Set.new,
+          closed: {
+          },
+          closing: {
+          },
           year_group_changes: {
           }
         }
@@ -48,13 +52,29 @@ def call(input_file:, **)
         new_schools = Set.new
 
         Zip::File.open(input_file) do |zip|
-          csv_entry = zip.glob("edubasealldata*.csv").first
-          csv_content = csv_entry.get_input_stream.read
+          links_csv = zip.glob("links_edubasealldata*.csv").first
+          links_csv_content = links_csv.get_input_stream.read
 
-          progress_bar = MavisCLI.progress_bar(csv_content.lines.count + 1)
+          @school_successors = {}
+          CSV.parse(
+            links_csv_content,
+            headers: true,
+            encoding: "ISO-8859-1:UTF-8"
+          ) do |row|
+            next unless row["LinkType"]&.include?("Successor")
+
+            @school_successors[row["URN"]] ||= []
+            @school_successors[row["URN"]] << row["LinkURN"]
+          end
+
+          school_data_csv = zip.glob("edubasealldata*.csv").first
+          school_csv_content = school_data_csv.get_input_stream.read
+
+          progress_bar =
+            MavisCLI.progress_bar(school_csv_content.lines.count + 1)
 
           CSV.parse(
-            csv_content,
+            school_csv_content,
             headers: true,
             encoding: "ISO-8859-1:UTF-8"
           ) do |row|
@@ -115,17 +135,29 @@ def call(input_file:, **)
             That have year group changes: #{schools_with_future_sessions[:year_group_changes].count} (#{schools_with_changed_year_groups_pct * 100}%)
         OUTPUT
 
-        puts <<~OUTPUT if schools_with_future_sessions[:closed].any?
-
-URNs of closed schools with future sessions:
-  #{schools_with_future_sessions[:closed].to_a.sort.join("\n  ")}
-          OUTPUT
-
-        puts <<~OUTPUT if schools_with_future_sessions[:closing].any?
+        if schools_with_future_sessions[:closed].any?
+          puts "\nURNs of closed schools with future sessions:"
+          schools_with_future_sessions[:closed].sort.each do |urn, successors|
+            if successors.any?
+              successor_info = format_successors_with_teams(successors)
+              puts "  #{urn} -> successor(s): #{successor_info}"
+            else
+              puts "  #{urn}"
+            end
+          end
+        end
 
-URNs of schools that will be closing, with future sessions:
-  #{schools_with_future_sessions[:closing].to_a.sort.join("\n  ")}
-          OUTPUT
+        if schools_with_future_sessions[:closing].any?
+          puts "\nURNs of schools that will be closing, with future sessions:"
+          schools_with_future_sessions[:closing].sort.each do |urn, successors|
+            if successors.any?
+              successor_info = format_successors_with_teams(successors)
+              puts "  #{urn} -> successor(s): #{successor_info}"
+            else
+              puts "  #{urn}"
+            end
+          end
+        end
 
         if schools_with_future_sessions[:year_group_changes].any?
           puts "\nURNs of schools with year group changes, with future sessions:"
@@ -141,6 +173,37 @@ def call(input_file:, **)
 
       private
 
+      def format_successors_with_teams(successor_urns)
+        successor_urns
+          .map do |successor_urn|
+            locations = Location.school.where(urn: successor_urn)
+
+            if locations.count == 1
+              teams = locations.sole.teams.uniq
+              if teams.any?
+                team_names = teams.map(&:name).join(", ")
+                "#{successor_urn} (Team: #{team_names})"
+              else
+                "#{successor_urn} (no team)"
+              end
+            elsif locations.count > 1
+              site_urns =
+                locations.where.not(site: nil).map(&:urn_and_site).join(", ")
+              team_names =
+                locations
+                  .where.not(site: nil)
+                  .flat_map(&:teams)
+                  .uniq
+                  .map(&:name)
+                  .join(", ")
+              "#{site_urns} (Teams: #{team_names})"
+            else
+              "#{successor_urn} (not found)"
+            end
+          end
+          .join(", ")
+      end
+
       def calculate_percentage(schools_set, metric)
         if schools_set[:existing].count.positive?
           schools_set[metric].count.to_f / schools_set[:existing].count
@@ -152,11 +215,12 @@ def calculate_percentage(schools_set, metric)
       def check_for_school_closure(row, school_set)
         urn = row["URN"]
         new_status = row["EstablishmentStatus (name)"]
+        successors = @school_successors[urn] || []
 
         if new_status == "Closed"
-          school_set[:closed] << urn
+          school_set[:closed][urn] = successors
         elsif new_status == "Open, but proposed to close"
-          school_set[:closing] << urn
+          school_set[:closing][urn] = successors
         end
       end
 

app/lib/mavis_cli/gias/download.rb

@@ -3,9 +3,10 @@
 # To get the latest version of the zip:
 # 1. Go to https://get-information-schools.service.gov.uk/Downloads
 # 2. Check "Establishment fields CSV"
-# 3. Submit
-# 4. Download the zip file
-# 5. Place it in db/data/dfe-schools.zip
+# 3. Check "Establishment links CSV"
+# 4. Submit
+# 5. Download the zip file
+# 6. Move the downloaded file to db/data/dfe-schools.zip
 #
 # Alternatively, you can run this task.
 
@@ -34,6 +35,9 @@ def call(output_file:, **)
         form = page.form_with(action: "/Downloads/Collate")
         form.checkbox_with(id: "establishment-fields-csv-checkbox").check
 
+        puts "Checking the establishment links CSV checkbox"
+        form.checkbox_with(id: "establishment-links-csv-checkbox").check
+
         puts "Submitting the form"
         download_page = form.submit
 

spec/features/cli_gias_check_import_spec.rb

@@ -32,6 +32,9 @@ def and_there_are_schools_with_future_sessions
         team: @team
       )
 
+    @successor_school =
+      create(:school, name: "The Aldgate Academy", urn: "100004", team: @team)
+
     @school2_with_future_session =
       create(
         :school,
@@ -90,7 +93,7 @@ def then_i_should_see_the_correct_counts
     expect(@output).to include("That have year group changes: 1")
 
     expect(@output).to include(
-      "URNs of closed schools with future sessions:\n  100000"
+      "URNs of closed schools with future sessions:\n  100000 -> successor(s): 100004 (Team: #{@team.name})"
     )
     expect(@output).to include(
       "URNs of schools that will be closing, with future sessions:\n  100002"

spec/features/cli_gias_import_spec.rb

@@ -45,7 +45,7 @@ def when_i_run_the_import_command
   end
 
   def then_schools_are_imported_correctly
-    expect(Location.count).to eq(6)
+    expect(Location.count).to eq(7)
     expect(Location.find_by_urn_and_site("100000").name).to eq(
       "The Aldgate School"
     )