791 - School field missing for users after school list update

Dockerfile

@@ -13,7 +13,7 @@ USER root
 RUN mkdir /isaac-logs
 RUN chmod 755 /isaac-logs
 RUN chown jetty /isaac-logs
-ADD resources/school_list_2025.tar.gz /local/data/
+ADD resources/schools_list_2025_december.tar.gz /local/data/
 COPY --from=base /isaac-api/target/isaac-api.war /var/lib/jetty/webapps/isaac-api.war
 RUN chmod 755 /var/lib/jetty/webapps/*
 RUN chown jetty /var/lib/jetty/webapps/*

compose-local-deps.yml

@@ -27,7 +27,7 @@ services:
     ports:
       - "9200:9200"
       - "9300:9300"
-    mem_limit: 6G
+    mem_limit: 8G
 
   cs-elasticsearch:
     network_mode: bridge
@@ -45,7 +45,7 @@ services:
     ports:
       - "9201:9200"
       - "9301:9300"
-    mem_limit: 6G
+    mem_limit: 8G
 
   postgres:
     network_mode: bridge

config-templates/content_indices.properties

@@ -1,4 +1,4 @@
 #
-#Mon Jan 28 2025
-latest=b75f3eed0165f039664aa89a6c7a5a6902e14e97
-live=b75f3eed0165f039664aa89a6c7a5a6902e14e97
+#Mon Jan 28 2026
+latest=49bbe52a1ab473f71e4e05a2ca1aed73fac87334
+live=49bbe52a1ab473f71e4e05a2ca1aed73fac87334

config-templates/linux-local-dev-segue-config.properties

@@ -19,7 +19,7 @@ EMAIL_SIGNATURE=Isaac Physics Project
 EVENT_ADMIN_EMAIL=events@isaacphysics.org
 EVENT_ICAL_UID_DOMAIN=isaacphysics.org
 
-SCHOOL_CSV_LIST_PATH=/local/data/school_list_2025.csv
+SCHOOL_CSV_LIST_PATH=/local/data/schools_list_2025_december.csv
 
 # Segue
 

config-templates/windows--local-dev-segue-config.properties

@@ -19,7 +19,7 @@ EMAIL_SIGNATURE=Isaac Physics Project
 EVENT_ADMIN_EMAIL=events@isaacphysics.org
 EVENT_ICAL_UID_DOMAIN=isaacphysics.org
 
-SCHOOL_CSV_LIST_PATH=C:\\dev\\isaac-other-resources\\school_list_2025.csv
+SCHOOL_CSV_LIST_PATH=C:\\dev\\isaac-other-resources\\schools_list_2025_december.csv
 
 # Segue
 

src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java

@@ -148,6 +148,7 @@ public final class Constants {
   /**
    * Enum to describe types of server environment / profile.
    */
+
   public enum EnvironmentType {
     PROD, DEV
   }

src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java

@@ -18,6 +18,7 @@
 
 import static uk.ac.cam.cl.dtg.segue.api.Constants.DEFAULT_RESULTS_LIMIT;
 import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOLS_INDEX_BASE;
+import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_CLOSED_FIELDNAME;
 import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_NAME_FIELDNAME_POJO;
 import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_POSTCODE_FIELDNAME_POJO;
 import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_URN_FIELDNAME;
@@ -31,6 +32,8 @@
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.api.client.util.Lists;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.inject.Inject;
 import java.io.IOException;
 import java.util.List;
@@ -65,18 +68,21 @@ public class SchoolListReader {
    */
   @Inject
   public SchoolListReader(final ISearchProvider searchProvider) {
+    log.info("Initializing SchoolListReader");
     this.searchProvider = searchProvider;
 
     String modificationDate;
     try {
       modificationDate = searchProvider.getById(
               SCHOOLS_INDEX_BASE, SchoolsIndexType.METADATA.toString(), "sourceFile").getSource().get("lastModified")
           .toString();
+      log.info("School list data source modification date: {}", modificationDate);
     } catch (SegueSearchException | ElasticsearchStatusException e) {
       log.error("Failed to retrieve school list modification date", e);
       modificationDate = "unknown";
     }
     dataSourceModificationDate = modificationDate;
+    log.info("SchoolListReader initialized successfully");
   }
 
   /**
@@ -88,35 +94,48 @@ public SchoolListReader(final ISearchProvider searchProvider) {
    */
   public List<School> findSchoolByNameOrPostCode(final String searchQuery)
       throws UnableToIndexSchoolsException, SegueSearchException {
+    if (log.isInfoEnabled()) {
+      log.info("School search initiated with query: {}", sanitiseExternalLogValue(searchQuery));
+    }
+
     if (!this.ensureSchoolList()) {
       log.error("Unable to ensure school search cache.");
       throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated");
     }
 
-    // FIXME: for one release cycle, we need backwards compatibility and so cannot use the fieldsThatMustMatch property
-    // It should be set to ImmutableMap.of("closed", ImmutableList.of("false"))
+    log.info("Executing fuzzy search with closed=false filter");
     List<String> schoolSearchResults = searchProvider.fuzzySearch(
         new BasicSearchParameters(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString(), 0,
             DEFAULT_RESULTS_LIMIT),
-        searchQuery, null, null, SCHOOL_URN_FIELDNAME_POJO, SCHOOL_NAME_FIELDNAME_POJO,
-        SCHOOL_POSTCODE_FIELDNAME_POJO
+        searchQuery,
+        ImmutableMap.of(SCHOOL_CLOSED_FIELDNAME, ImmutableList.of("false")),
+        null,
+        SCHOOL_URN_FIELDNAME_POJO, SCHOOL_NAME_FIELDNAME_POJO, SCHOOL_POSTCODE_FIELDNAME_POJO
     ).getResults();
 
+    if (log.isInfoEnabled()) {
+      log.info("Elasticsearch returned {} results for query: {}", schoolSearchResults.size(),
+          sanitiseExternalLogValue(searchQuery));
+    }
+
     List<School> resultList = Lists.newArrayList();
+    int parseErrors = 0;
     for (String schoolString : schoolSearchResults) {
       try {
-        School school = mapper.readValue(schoolString, School.class);
-        if (school.isClosed() != null && school.isClosed()) {
-          // FIXME: this filtering will be unnecessary once the above fix is implemented!
-          continue;
-        }
-        resultList.add(school);
+        resultList.add(mapper.readValue(schoolString, School.class));
       } catch (JsonParseException | JsonMappingException e) {
         log.error("Unable to parse the school {}", schoolString, e);
+        parseErrors++;
       } catch (IOException e) {
         log.error("IOException {}", schoolString, e);
+        parseErrors++;
       }
     }
+
+    if (log.isInfoEnabled()) {
+      log.info("School search completed. Query: {}, Results: {}, Parse errors: {}",
+          sanitiseExternalLogValue(searchQuery), resultList.size(), parseErrors);
+    }
     return resultList;
   }
 
@@ -132,12 +151,18 @@ public List<School> findSchoolByNameOrPostCode(final String searchQuery)
    */
   public School findSchoolById(final String schoolURN) throws UnableToIndexSchoolsException, JsonParseException,
       JsonMappingException, IOException, SegueSearchException {
+    if (log.isInfoEnabled()) {
+      log.info("School lookup by URN initiated: {}", sanitiseExternalLogValue(schoolURN));
+    }
 
     if (!this.ensureSchoolList()) {
       log.error("Unable to ensure school search cache.");
       throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated");
     }
 
+    if (log.isInfoEnabled()) {
+      log.info("Executing exact match search for URN: {}", sanitiseExternalLogValue(schoolURN));
+    }
     List<String> matchingSchoolList;
 
     matchingSchoolList = searchProvider.findByExactMatch(
@@ -146,6 +171,9 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools
         SCHOOL_URN_FIELDNAME.toLowerCase() + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, schoolURN, null).getResults();
 
     if (matchingSchoolList.isEmpty()) {
+      if (log.isInfoEnabled()) {
+        log.info("School lookup completed. URN: {} not found in index", sanitiseExternalLogValue(schoolURN));
+      }
       return null;
     }
 
@@ -154,7 +182,12 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools
           + sanitiseExternalLogValue(schoolURN) + " results: " + matchingSchoolList);
     }
 
-    return mapper.readValue(matchingSchoolList.get(0), School.class);
+    School school = mapper.readValue(matchingSchoolList.get(0), School.class);
+    if (log.isInfoEnabled()) {
+      log.info("School lookup completed. URN: {}, Found: {}, Closed: {}",
+          sanitiseExternalLogValue(schoolURN), school.getName(), school.isClosed());
+    }
+    return school;
   }
 
 
@@ -164,7 +197,11 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools
    * @return true if we have an index or false if not. If false we cannot guarantee a response.
    */
   private boolean ensureSchoolList() {
-    return searchProvider.hasIndex(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString());
+    boolean indexExists = searchProvider.hasIndex(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString());
+    if (!indexExists) {
+      log.warn("School index not found: {}/{}", SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH);
+    }
+    return indexExists;
   }
 
 

src/test/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReaderTest.java

@@ -0,0 +1,165 @@
+package uk.ac.cam.cl.dtg.segue.dao.schools;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_CLOSED_FIELDNAME;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import uk.ac.cam.cl.dtg.isaac.dos.users.School;
+import uk.ac.cam.cl.dtg.isaac.dto.ResultsWrapper;
+import uk.ac.cam.cl.dtg.segue.search.BasicSearchParameters;
+import uk.ac.cam.cl.dtg.segue.search.ISearchProvider;
+import uk.ac.cam.cl.dtg.segue.search.SegueSearchException;
+
+class SchoolListReaderTest {
+
+  private ISearchProvider mockSearchProvider;
+  private ObjectMapper objectMapper;
+
+  private School openSchool;
+  private School closedSchool;
+  private String openSchoolJson;
+  private String closedSchoolJson;
+
+  @BeforeEach
+  void setUp() throws JsonProcessingException, SegueSearchException {
+    mockSearchProvider = createNiceMock(ISearchProvider.class);
+    objectMapper = new ObjectMapper();
+
+    openSchool = new School("100001", "Open Academy", "AB1 2CD", false, School.SchoolDataSource.GOVERNMENT_UK);
+    closedSchool = new School("100002", "Closed Grammar School", "XY9 8ZZ", true, School.SchoolDataSource.GOVERNMENT_UK);
+
+    openSchoolJson = objectMapper.writeValueAsString(openSchool);
+    closedSchoolJson = objectMapper.writeValueAsString(closedSchool);
+
+    expect(mockSearchProvider.getById(anyString(), anyString(), anyString()))
+        .andThrow(new SegueSearchException("Test - metadata not available")).anyTimes();
+    expect(mockSearchProvider.hasIndex(anyString(), anyString()))
+        .andReturn(true).anyTimes();
+  }
+
+  @Test
+  void findSchoolByNameOrPostCode_shouldFilterClosedSchools() throws Exception {
+    // Expect fuzzySearch to be called with closed=false filter
+    Map<String, List<String>> expectedFilter = ImmutableMap.of(SCHOOL_CLOSED_FIELDNAME, ImmutableList.of("false"));
+
+    expect(mockSearchProvider.fuzzySearch(
+        anyObject(BasicSearchParameters.class),
+        eq("test query"),
+        eq(expectedFilter),
+        anyObject(),
+        anyString(), anyString(), anyString()
+    )).andReturn(new ResultsWrapper<>(ImmutableList.of(openSchoolJson), 1L));
+
+    replay(mockSearchProvider);
+
+    SchoolListReader reader = new SchoolListReader(mockSearchProvider);
+    List<School> results = reader.findSchoolByNameOrPostCode("test query");
+
+    verify(mockSearchProvider);
+    assertEquals(1, results.size());
+    assertEquals("100001", results.get(0).getUrn());
+    assertEquals("Open Academy", results.get(0).getName());
+    assertEquals(false, results.get(0).isClosed());
+  }
+
+  @Test
+  void findSchoolByNameOrPostCode_shouldReturnEmptyListWhenNoOpenSchoolsMatch() throws Exception {
+    // Elasticsearch already filters, so empty results returned
+    Map<String, List<String>> expectedFilter = ImmutableMap.of(SCHOOL_CLOSED_FIELDNAME, ImmutableList.of("false"));
+
+    expect(mockSearchProvider.fuzzySearch(
+        anyObject(BasicSearchParameters.class),
+        eq("closed school query"),
+        eq(expectedFilter),
+        anyObject(),
+        anyString(), anyString(), anyString()
+    )).andReturn(new ResultsWrapper<>(Collections.emptyList(), 0L));
+
+    replay(mockSearchProvider);
+
+    SchoolListReader reader = new SchoolListReader(mockSearchProvider);
+    List<School> results = reader.findSchoolByNameOrPostCode("closed school query");
+
+    verify(mockSearchProvider);
+    assertTrue(results.isEmpty());
+  }
+
+  @Test
+  void findSchoolById_shouldReturnClosedSchool() throws Exception {
+    // findByExactMatch should NOT filter by closed status
+    expect(mockSearchProvider.findByExactMatch(
+        anyObject(BasicSearchParameters.class),
+        anyString(),
+        eq("100002"),
+        anyObject()
+    )).andReturn(new ResultsWrapper<>(ImmutableList.of(closedSchoolJson), 1L));
+
+    replay(mockSearchProvider);
+
+    SchoolListReader reader = new SchoolListReader(mockSearchProvider);
+    School result = reader.findSchoolById("100002");
+
+    verify(mockSearchProvider);
+    assertNotNull(result);
+    assertEquals("100002", result.getUrn());
+    assertEquals("Closed Grammar School", result.getName());
+    assertEquals(true, result.isClosed());
+  }
+
+  @Test
+  void findSchoolById_shouldReturnOpenSchool() throws Exception {
+    expect(mockSearchProvider.findByExactMatch(
+        anyObject(BasicSearchParameters.class),
+        anyString(),
+        eq("100001"),
+        anyObject()
+    )).andReturn(new ResultsWrapper<>(ImmutableList.of(openSchoolJson), 1L));
+
+    replay(mockSearchProvider);
+
+    SchoolListReader reader = new SchoolListReader(mockSearchProvider);
+    School result = reader.findSchoolById("100001");
+
+    verify(mockSearchProvider);
+    assertNotNull(result);
+    assertEquals("100001", result.getUrn());
+    assertEquals("Open Academy", result.getName());
+    assertEquals(false, result.isClosed());
+  }
+
+  @Test
+  void findSchoolById_shouldReturnNullWhenSchoolNotFound() throws Exception {
+    expect(mockSearchProvider.findByExactMatch(
+        anyObject(BasicSearchParameters.class),
+        anyString(),
+        eq("999999"),
+        anyObject()
+    )).andReturn(new ResultsWrapper<>(Collections.emptyList(), 0L));
+
+    replay(mockSearchProvider);
+
+    SchoolListReader reader = new SchoolListReader(mockSearchProvider);
+    School result = reader.findSchoolById("999999");
+
+    verify(mockSearchProvider);
+    assertNull(result);
+  }
+}

src/test/resources/segue-integration-test-config.properties

@@ -20,7 +20,7 @@ EVENT_ADMIN_EMAIL=events@isaaccomputerscience.org
 EVENT_ICAL_UID_DOMAIN=isaaccomputerscience.org
 EVENT_GROUP_RESERVATION_LIMIT=69
 
-SCHOOL_CSV_LIST_PATH=src/test/resources/school_list_2025.csv
+SCHOOL_CSV_LIST_PATH=src/test/resources/schools_list_2025_december.csv
 
 # Segue