diff --git a/Dockerfile b/Dockerfile index ec092408d2..1e6eaff04e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ USER root RUN mkdir /isaac-logs RUN chmod 755 /isaac-logs RUN chown jetty /isaac-logs -ADD resources/school_list_2025.tar.gz /local/data/ +ADD resources/schools_list_2025_december.tar.gz /local/data/ COPY --from=base /isaac-api/target/isaac-api.war /var/lib/jetty/webapps/isaac-api.war RUN chmod 755 /var/lib/jetty/webapps/* RUN chown jetty /var/lib/jetty/webapps/* diff --git a/compose-local-deps.yml b/compose-local-deps.yml index 87fa76ebdd..d126100761 100644 --- a/compose-local-deps.yml +++ b/compose-local-deps.yml @@ -27,7 +27,7 @@ services: ports: - "9200:9200" - "9300:9300" - mem_limit: 6G + mem_limit: 8G cs-elasticsearch: network_mode: bridge @@ -45,7 +45,7 @@ services: ports: - "9201:9200" - "9301:9300" - mem_limit: 6G + mem_limit: 8G postgres: network_mode: bridge diff --git a/config-templates/content_indices.properties b/config-templates/content_indices.properties index 678deaaed8..3e58a6a248 100644 --- a/config-templates/content_indices.properties +++ b/config-templates/content_indices.properties @@ -1,4 +1,4 @@ # -#Mon Jan 28 2025 -latest=b75f3eed0165f039664aa89a6c7a5a6902e14e97 -live=b75f3eed0165f039664aa89a6c7a5a6902e14e97 +#Mon Jan 28 2026 +latest=49bbe52a1ab473f71e4e05a2ca1aed73fac87334 +live=49bbe52a1ab473f71e4e05a2ca1aed73fac87334 diff --git a/config-templates/linux-local-dev-segue-config.properties b/config-templates/linux-local-dev-segue-config.properties index c383e65107..9d8bca0d93 100755 --- a/config-templates/linux-local-dev-segue-config.properties +++ b/config-templates/linux-local-dev-segue-config.properties @@ -19,7 +19,7 @@ EMAIL_SIGNATURE=Isaac Physics Project EVENT_ADMIN_EMAIL=events@isaacphysics.org EVENT_ICAL_UID_DOMAIN=isaacphysics.org -SCHOOL_CSV_LIST_PATH=/local/data/school_list_2025.csv +SCHOOL_CSV_LIST_PATH=/local/data/schools_list_2025_december.csv # Segue diff --git a/config-templates/windows--local-dev-segue-config.properties b/config-templates/windows--local-dev-segue-config.properties index 64087eb82f..1f2cd430da 100644 --- a/config-templates/windows--local-dev-segue-config.properties +++ b/config-templates/windows--local-dev-segue-config.properties @@ -19,7 +19,7 @@ EMAIL_SIGNATURE=Isaac Physics Project EVENT_ADMIN_EMAIL=events@isaacphysics.org EVENT_ICAL_UID_DOMAIN=isaacphysics.org -SCHOOL_CSV_LIST_PATH=C:\\dev\\isaac-other-resources\\school_list_2025.csv +SCHOOL_CSV_LIST_PATH=C:\\dev\\isaac-other-resources\\schools_list_2025_december.csv # Segue diff --git a/resources/school_list_2024.tar.gz b/resources/school_list_2024.tar.gz deleted file mode 100644 index 0197b16310..0000000000 Binary files a/resources/school_list_2024.tar.gz and /dev/null differ diff --git a/resources/school_list_2025.tar.gz b/resources/school_list_2025.tar.gz deleted file mode 100644 index 423b422ddd..0000000000 Binary files a/resources/school_list_2025.tar.gz and /dev/null differ diff --git a/resources/schools_list_2025_december.tar.gz b/resources/schools_list_2025_december.tar.gz new file mode 100644 index 0000000000..a387a6627e Binary files /dev/null and b/resources/schools_list_2025_december.tar.gz differ diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java index 7cc337d1f5..3bb40e0806 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/api/Constants.java @@ -148,6 +148,7 @@ public final class Constants { /** * Enum to describe types of server environment / profile. */ + public enum EnvironmentType { PROD, DEV } diff --git a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java index 88589b5b17..8b722c4d09 100644 --- a/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java +++ b/src/main/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReader.java @@ -18,6 +18,7 @@ import static uk.ac.cam.cl.dtg.segue.api.Constants.DEFAULT_RESULTS_LIMIT; import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOLS_INDEX_BASE; +import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_CLOSED_FIELDNAME; import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_NAME_FIELDNAME_POJO; import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_POSTCODE_FIELDNAME_POJO; import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_URN_FIELDNAME; @@ -31,6 +32,8 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.util.Lists; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import java.io.IOException; import java.util.List; @@ -65,6 +68,7 @@ public class SchoolListReader { */ @Inject public SchoolListReader(final ISearchProvider searchProvider) { + log.info("Initializing SchoolListReader"); this.searchProvider = searchProvider; String modificationDate; @@ -72,11 +76,13 @@ public SchoolListReader(final ISearchProvider searchProvider) { modificationDate = searchProvider.getById( SCHOOLS_INDEX_BASE, SchoolsIndexType.METADATA.toString(), "sourceFile").getSource().get("lastModified") .toString(); + log.info("School list data source modification date: {}", modificationDate); } catch (SegueSearchException | ElasticsearchStatusException e) { log.error("Failed to retrieve school list modification date", e); modificationDate = "unknown"; } dataSourceModificationDate = modificationDate; + log.info("SchoolListReader initialized successfully"); } /** @@ -88,35 +94,48 @@ public SchoolListReader(final ISearchProvider searchProvider) { */ public List findSchoolByNameOrPostCode(final String searchQuery) throws UnableToIndexSchoolsException, SegueSearchException { + if (log.isInfoEnabled()) { + log.info("School search initiated with query: {}", sanitiseExternalLogValue(searchQuery)); + } + if (!this.ensureSchoolList()) { log.error("Unable to ensure school search cache."); throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated"); } - // FIXME: for one release cycle, we need backwards compatibility and so cannot use the fieldsThatMustMatch property - // It should be set to ImmutableMap.of("closed", ImmutableList.of("false")) + log.info("Executing fuzzy search with closed=false filter"); List schoolSearchResults = searchProvider.fuzzySearch( new BasicSearchParameters(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString(), 0, DEFAULT_RESULTS_LIMIT), - searchQuery, null, null, SCHOOL_URN_FIELDNAME_POJO, SCHOOL_NAME_FIELDNAME_POJO, - SCHOOL_POSTCODE_FIELDNAME_POJO + searchQuery, + ImmutableMap.of(SCHOOL_CLOSED_FIELDNAME, ImmutableList.of("false")), + null, + SCHOOL_URN_FIELDNAME_POJO, SCHOOL_NAME_FIELDNAME_POJO, SCHOOL_POSTCODE_FIELDNAME_POJO ).getResults(); + if (log.isInfoEnabled()) { + log.info("Elasticsearch returned {} results for query: {}", schoolSearchResults.size(), + sanitiseExternalLogValue(searchQuery)); + } + List resultList = Lists.newArrayList(); + int parseErrors = 0; for (String schoolString : schoolSearchResults) { try { - School school = mapper.readValue(schoolString, School.class); - if (school.isClosed() != null && school.isClosed()) { - // FIXME: this filtering will be unnecessary once the above fix is implemented! - continue; - } - resultList.add(school); + resultList.add(mapper.readValue(schoolString, School.class)); } catch (JsonParseException | JsonMappingException e) { log.error("Unable to parse the school {}", schoolString, e); + parseErrors++; } catch (IOException e) { log.error("IOException {}", schoolString, e); + parseErrors++; } } + + if (log.isInfoEnabled()) { + log.info("School search completed. Query: {}, Results: {}, Parse errors: {}", + sanitiseExternalLogValue(searchQuery), resultList.size(), parseErrors); + } return resultList; } @@ -132,12 +151,18 @@ public List findSchoolByNameOrPostCode(final String searchQuery) */ public School findSchoolById(final String schoolURN) throws UnableToIndexSchoolsException, JsonParseException, JsonMappingException, IOException, SegueSearchException { + if (log.isInfoEnabled()) { + log.info("School lookup by URN initiated: {}", sanitiseExternalLogValue(schoolURN)); + } if (!this.ensureSchoolList()) { log.error("Unable to ensure school search cache."); throw new UnableToIndexSchoolsException("unable to ensure the cache has been populated"); } + if (log.isInfoEnabled()) { + log.info("Executing exact match search for URN: {}", sanitiseExternalLogValue(schoolURN)); + } List matchingSchoolList; matchingSchoolList = searchProvider.findByExactMatch( @@ -146,6 +171,9 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools SCHOOL_URN_FIELDNAME.toLowerCase() + "." + UNPROCESSED_SEARCH_FIELD_SUFFIX, schoolURN, null).getResults(); if (matchingSchoolList.isEmpty()) { + if (log.isInfoEnabled()) { + log.info("School lookup completed. URN: {} not found in index", sanitiseExternalLogValue(schoolURN)); + } return null; } @@ -154,7 +182,12 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools + sanitiseExternalLogValue(schoolURN) + " results: " + matchingSchoolList); } - return mapper.readValue(matchingSchoolList.get(0), School.class); + School school = mapper.readValue(matchingSchoolList.get(0), School.class); + if (log.isInfoEnabled()) { + log.info("School lookup completed. URN: {}, Found: {}, Closed: {}", + sanitiseExternalLogValue(schoolURN), school.getName(), school.isClosed()); + } + return school; } @@ -164,7 +197,11 @@ public School findSchoolById(final String schoolURN) throws UnableToIndexSchools * @return true if we have an index or false if not. If false we cannot guarantee a response. */ private boolean ensureSchoolList() { - return searchProvider.hasIndex(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString()); + boolean indexExists = searchProvider.hasIndex(SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH.toString()); + if (!indexExists) { + log.warn("School index not found: {}/{}", SCHOOLS_INDEX_BASE, SchoolsIndexType.SCHOOL_SEARCH); + } + return indexExists; } diff --git a/src/test/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReaderTest.java b/src/test/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReaderTest.java new file mode 100644 index 0000000000..e3132720b5 --- /dev/null +++ b/src/test/java/uk/ac/cam/cl/dtg/segue/dao/schools/SchoolListReaderTest.java @@ -0,0 +1,165 @@ +package uk.ac.cam.cl.dtg.segue.dao.schools; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static uk.ac.cam.cl.dtg.segue.api.Constants.SCHOOL_CLOSED_FIELDNAME; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import uk.ac.cam.cl.dtg.isaac.dos.users.School; +import uk.ac.cam.cl.dtg.isaac.dto.ResultsWrapper; +import uk.ac.cam.cl.dtg.segue.search.BasicSearchParameters; +import uk.ac.cam.cl.dtg.segue.search.ISearchProvider; +import uk.ac.cam.cl.dtg.segue.search.SegueSearchException; + +class SchoolListReaderTest { + + private ISearchProvider mockSearchProvider; + private ObjectMapper objectMapper; + + private School openSchool; + private School closedSchool; + private String openSchoolJson; + private String closedSchoolJson; + + @BeforeEach + void setUp() throws JsonProcessingException, SegueSearchException { + mockSearchProvider = createNiceMock(ISearchProvider.class); + objectMapper = new ObjectMapper(); + + openSchool = new School("100001", "Open Academy", "AB1 2CD", false, School.SchoolDataSource.GOVERNMENT_UK); + closedSchool = new School("100002", "Closed Grammar School", "XY9 8ZZ", true, School.SchoolDataSource.GOVERNMENT_UK); + + openSchoolJson = objectMapper.writeValueAsString(openSchool); + closedSchoolJson = objectMapper.writeValueAsString(closedSchool); + + expect(mockSearchProvider.getById(anyString(), anyString(), anyString())) + .andThrow(new SegueSearchException("Test - metadata not available")).anyTimes(); + expect(mockSearchProvider.hasIndex(anyString(), anyString())) + .andReturn(true).anyTimes(); + } + + @Test + void findSchoolByNameOrPostCode_shouldFilterClosedSchools() throws Exception { + // Expect fuzzySearch to be called with closed=false filter + Map> expectedFilter = ImmutableMap.of(SCHOOL_CLOSED_FIELDNAME, ImmutableList.of("false")); + + expect(mockSearchProvider.fuzzySearch( + anyObject(BasicSearchParameters.class), + eq("test query"), + eq(expectedFilter), + anyObject(), + anyString(), anyString(), anyString() + )).andReturn(new ResultsWrapper<>(ImmutableList.of(openSchoolJson), 1L)); + + replay(mockSearchProvider); + + SchoolListReader reader = new SchoolListReader(mockSearchProvider); + List results = reader.findSchoolByNameOrPostCode("test query"); + + verify(mockSearchProvider); + assertEquals(1, results.size()); + assertEquals("100001", results.get(0).getUrn()); + assertEquals("Open Academy", results.get(0).getName()); + assertEquals(false, results.get(0).isClosed()); + } + + @Test + void findSchoolByNameOrPostCode_shouldReturnEmptyListWhenNoOpenSchoolsMatch() throws Exception { + // Elasticsearch already filters, so empty results returned + Map> expectedFilter = ImmutableMap.of(SCHOOL_CLOSED_FIELDNAME, ImmutableList.of("false")); + + expect(mockSearchProvider.fuzzySearch( + anyObject(BasicSearchParameters.class), + eq("closed school query"), + eq(expectedFilter), + anyObject(), + anyString(), anyString(), anyString() + )).andReturn(new ResultsWrapper<>(Collections.emptyList(), 0L)); + + replay(mockSearchProvider); + + SchoolListReader reader = new SchoolListReader(mockSearchProvider); + List results = reader.findSchoolByNameOrPostCode("closed school query"); + + verify(mockSearchProvider); + assertTrue(results.isEmpty()); + } + + @Test + void findSchoolById_shouldReturnClosedSchool() throws Exception { + // findByExactMatch should NOT filter by closed status + expect(mockSearchProvider.findByExactMatch( + anyObject(BasicSearchParameters.class), + anyString(), + eq("100002"), + anyObject() + )).andReturn(new ResultsWrapper<>(ImmutableList.of(closedSchoolJson), 1L)); + + replay(mockSearchProvider); + + SchoolListReader reader = new SchoolListReader(mockSearchProvider); + School result = reader.findSchoolById("100002"); + + verify(mockSearchProvider); + assertNotNull(result); + assertEquals("100002", result.getUrn()); + assertEquals("Closed Grammar School", result.getName()); + assertEquals(true, result.isClosed()); + } + + @Test + void findSchoolById_shouldReturnOpenSchool() throws Exception { + expect(mockSearchProvider.findByExactMatch( + anyObject(BasicSearchParameters.class), + anyString(), + eq("100001"), + anyObject() + )).andReturn(new ResultsWrapper<>(ImmutableList.of(openSchoolJson), 1L)); + + replay(mockSearchProvider); + + SchoolListReader reader = new SchoolListReader(mockSearchProvider); + School result = reader.findSchoolById("100001"); + + verify(mockSearchProvider); + assertNotNull(result); + assertEquals("100001", result.getUrn()); + assertEquals("Open Academy", result.getName()); + assertEquals(false, result.isClosed()); + } + + @Test + void findSchoolById_shouldReturnNullWhenSchoolNotFound() throws Exception { + expect(mockSearchProvider.findByExactMatch( + anyObject(BasicSearchParameters.class), + anyString(), + eq("999999"), + anyObject() + )).andReturn(new ResultsWrapper<>(Collections.emptyList(), 0L)); + + replay(mockSearchProvider); + + SchoolListReader reader = new SchoolListReader(mockSearchProvider); + School result = reader.findSchoolById("999999"); + + verify(mockSearchProvider); + assertNull(result); + } +} diff --git a/src/test/resources/segue-integration-test-config.properties b/src/test/resources/segue-integration-test-config.properties index c2fe6939f5..5315f784fe 100755 --- a/src/test/resources/segue-integration-test-config.properties +++ b/src/test/resources/segue-integration-test-config.properties @@ -20,7 +20,7 @@ EVENT_ADMIN_EMAIL=events@isaaccomputerscience.org EVENT_ICAL_UID_DOMAIN=isaaccomputerscience.org EVENT_GROUP_RESERVATION_LIMIT=69 -SCHOOL_CSV_LIST_PATH=src/test/resources/school_list_2025.csv +SCHOOL_CSV_LIST_PATH=src/test/resources/schools_list_2025_december.csv # Segue