From 2a4a27c079a2e398a250968e406de54db74a8715 Mon Sep 17 00:00:00 2001 From: John Kallies <3021949+JohnKallies@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:52:08 -0500 Subject: [PATCH 1/2] Add controller for ElasticSearch admin --- .../main/kotlin/edu/wgu/osmt/RoutePaths.kt | 31 ++++++----- .../wgu/osmt/collection/CollectionEsRepo.kt | 6 +++ .../ElasticSearchAdminController.kt | 53 +++++++++++++++++++ .../elasticsearch/ElasticSearchReindexer.kt | 23 +++++--- .../wgu/osmt/elasticsearch/ReindexCommand.kt | 2 +- .../edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 6 +++ .../edu/wgu/osmt/keyword/KeywordEsRepo.kt | 6 +++ .../edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 7 +++ 8 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt diff --git a/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt b/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt index 240d6f4af..322c95199 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/RoutePaths.kt @@ -9,36 +9,39 @@ object RoutePaths { const val SEARCH_SIMILARITIES = "$SEARCH_SKILLS/similarities" const val SEARCH_COLLECTIONS = "$SEARCH_PATH/collections" - const val SKILLS_PATH = "/api/skills" + const val SKILLS_PATH = "$API/skills" const val SKILLS_LIST = SKILLS_PATH const val SKILLS_CREATE = SKILLS_PATH const val SKILL_PUBLISH = "$SKILLS_PATH/publish" const val SKILL_DETAIL = "$SKILLS_PATH/{uuid}" const val SKILL_UPDATE = "$SKILL_DETAIL/update" - const val SKILL_AUDIT_LOG = "${SKILL_DETAIL}/log" + const val SKILL_AUDIT_LOG = "$SKILL_DETAIL/log" - const val COLLECTIONS_PATH = "/api/collections" + const val COLLECTIONS_PATH = "$API/collections" const val COLLECTIONS_LIST = COLLECTIONS_PATH const val COLLECTION_CREATE = COLLECTIONS_PATH const val COLLECTION_PUBLISH = "$COLLECTIONS_PATH/publish" - const val COLLECTION_DETAIL = "${COLLECTIONS_PATH}/{uuid}" - const val COLLECTION_UPDATE = "${COLLECTION_DETAIL}/update" - const val COLLECTION_SKILLS_UPDATE = "${COLLECTION_DETAIL}/updateSkills" - const val COLLECTION_SKILLS = "${COLLECTION_DETAIL}/skills" - const val COLLECTION_AUDIT_LOG = "${COLLECTION_DETAIL}/log" - const val COLLECTION_CSV = "${COLLECTION_DETAIL}/csv" + const val COLLECTION_DETAIL = "$COLLECTIONS_PATH/{uuid}" + const val COLLECTION_UPDATE = "$COLLECTION_DETAIL/update" + const val COLLECTION_SKILLS_UPDATE = "$COLLECTION_DETAIL/updateSkills" + const val COLLECTION_SKILLS = "$COLLECTION_DETAIL/skills" + const val COLLECTION_AUDIT_LOG = "$COLLECTION_DETAIL/log" + const val COLLECTION_CSV = "$COLLECTION_DETAIL/csv" - const val TASKS_PATH = "/api/results" - const val TASK_DETAIL_TEXT = "${TASKS_PATH}/text/{uuid}" - const val TASK_DETAIL_BATCH = "${TASKS_PATH}/batch/{uuid}" - const val TASK_DETAIL_SKILLS = "${TASKS_PATH}/skills/{uuid}" + const val TASKS_PATH = "$API/results" + const val TASK_DETAIL_TEXT = "$TASKS_PATH/text/{uuid}" + const val TASK_DETAIL_BATCH = "$TASKS_PATH/batch/{uuid}" + const val TASK_DETAIL_SKILLS = "$TASKS_PATH/skills/{uuid}" const val SEARCH_JOBCODES_PATH = "$SEARCH_PATH/jobcodes" - const val SEARCH_KEYWORDS_PATH = "$SEARCH_PATH/keywords" + const val ES_ADMIN = "$API/es-admin" + const val ES_ADMIN_DELETE_INDICES = "$ES_ADMIN/delete-indices" + const val ES_ADMIN_REINDEX = "$ES_ADMIN/reindex" + object QueryParams { const val FROM = "from" const val SIZE = "size" diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index 4373fa46a..f8c343373 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -2,6 +2,7 @@ package edu.wgu.osmt.collection import edu.wgu.osmt.PaginationDefaults import edu.wgu.osmt.api.model.ApiSearch +import edu.wgu.osmt.config.INDEX_COLLECTION_DOC import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.FindsAllByPublishStatus import edu.wgu.osmt.richskill.RichSkillDoc @@ -16,6 +17,7 @@ import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -34,6 +36,10 @@ interface CustomCollectionQueries : FindsAllByPublishStatus { Sort.by("name.keyword").descending() ) ): SearchHits + + fun deleteIndex() { + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_COLLECTION_DOC)).delete(); + } } class CustomCollectionQueriesImpl @Autowired constructor( diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt new file mode 100644 index 000000000..510160c9b --- /dev/null +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt @@ -0,0 +1,53 @@ +package edu.wgu.osmt.elasticsearch + +import edu.wgu.osmt.RoutePaths.ES_ADMIN_DELETE_INDICES +import edu.wgu.osmt.RoutePaths.ES_ADMIN_REINDEX +import edu.wgu.osmt.config.AppConfig +import edu.wgu.osmt.security.OAuthHelper +import edu.wgu.osmt.task.* +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.* +import org.springframework.stereotype.Controller +import org.springframework.transaction.annotation.Transactional +import org.springframework.web.bind.annotation.* +import org.springframework.web.server.ResponseStatusException +import java.util.concurrent.ForkJoinPool + +@Controller +@Transactional +class ElasticSearchAdminController @Autowired constructor( + val appConfig: AppConfig, + val oAuthHelper: OAuthHelper, + val esReindexer: ElasticSearchReindexer +) { + + @RequestMapping(ES_ADMIN_DELETE_INDICES) + @PostMapping + fun deleteElasticSearchIndices(): ResponseEntity { + + if (!oAuthHelper.hasRole(appConfig.roleAdmin)) { + throw ResponseStatusException(HttpStatus.UNAUTHORIZED) + } + + ForkJoinPool.commonPool().submit(esReindexer::deleteAllIndices) + return ResponseEntity( + "Deleting ElasticSearch indices in the background. Please refer to the logs.", + HttpStatus.ACCEPTED + ) + } + + @RequestMapping(ES_ADMIN_REINDEX) + @PostMapping + fun reindexElasticSearch(): ResponseEntity { + + if (!oAuthHelper.hasRole(appConfig.roleAdmin)) { + throw ResponseStatusException(HttpStatus.UNAUTHORIZED) + } + + ForkJoinPool.commonPool().submit(esReindexer::reindexAll) + return ResponseEntity( + "Reindexing ElasticSearch in the background. Please refer to the logs.", + HttpStatus.ACCEPTED + ) + } +} diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchReindexer.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchReindexer.kt index 04957ad72..f523a2519 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchReindexer.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchReindexer.kt @@ -51,14 +51,21 @@ class ElasticSearchReindexer { @Value("\${edu.wgu.osmt.elasticsearch.Reindex.batch_size:1000}") lateinit var limit: Integer + fun deleteAllIndices() { + richSkillEsRepo.deleteIndex() + collectionEsRepo.deleteIndex() + keywordEsRepo.deleteIndex() + jobCodeEsRepo.deleteIndex() + } + fun reindexAll() { - reimportSkills() - reimportCollections() - reimportKeywords() - reimportJobCodes() + reindexSkills() + reindexCollections() + reindexKeywords() + reindexJobCodes() } - private fun reimportSkills() { + private fun reindexSkills() { var page = 0 var exit = false val trace = ProcessLogger("skills", logger) @@ -82,7 +89,7 @@ class ElasticSearchReindexer { } } - private fun reimportCollections() { + private fun reindexCollections() { var page = 0 var exit = false var trace = ProcessLogger("collections", logger) @@ -106,7 +113,7 @@ class ElasticSearchReindexer { } } - private fun reimportKeywords() { + private fun reindexKeywords() { var page = 0 var exit = false var trace = ProcessLogger("keywords", logger) @@ -130,7 +137,7 @@ class ElasticSearchReindexer { } } - private fun reimportJobCodes() { + private fun reindexJobCodes() { var page = 0 var exit = false var trace = ProcessLogger("jobCodes", logger) diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ReindexCommand.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ReindexCommand.kt index eecb692dc..b883696ab 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ReindexCommand.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ReindexCommand.kt @@ -22,10 +22,10 @@ class ReindexCommand: CommandLineRunner { private lateinit var applicationContext: ApplicationContext override fun run(vararg args: String?) { + elasticSearchReindexer.deleteAllIndices() elasticSearchReindexer.reindexAll() (applicationContext as ConfigurableApplicationContext).close() } - } fun main(args: Array) { diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 3c84d7a63..59d422192 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -1,5 +1,6 @@ package edu.wgu.osmt.jobcode +import edu.wgu.osmt.config.INDEX_JOBCODE_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.Operator @@ -9,6 +10,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -16,6 +18,10 @@ import org.springframework.data.elasticsearch.repository.config.EnableElasticsea interface CustomJobCodeRepository { val elasticSearchTemplate: ElasticsearchRestTemplate fun typeAheadSearch(query: String): SearchHits + + fun deleteIndex() { + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_JOBCODE_DOC)).delete(); + } } class CustomJobCodeRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 434042682..5629828c0 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -1,5 +1,6 @@ package edu.wgu.osmt.keyword +import edu.wgu.osmt.config.INDEX_KEYWORD_DOC import edu.wgu.osmt.elasticsearch.OffsetPageable import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.sort.SortBuilders @@ -7,6 +8,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Configuration import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -14,6 +16,10 @@ import org.springframework.data.elasticsearch.repository.config.EnableElasticsea interface CustomKeywordRepository { val elasticSearchTemplate: ElasticsearchRestTemplate fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits + + fun deleteIndex() { + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_KEYWORD_DOC)).delete(); + } } class CustomKeywordRepositoryImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 3c4522fb6..4dc8c38fc 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -4,6 +4,7 @@ import edu.wgu.osmt.PaginationDefaults import edu.wgu.osmt.api.model.ApiAdvancedSearch import edu.wgu.osmt.api.model.ApiSearch import edu.wgu.osmt.api.model.ApiSimilaritySearch +import edu.wgu.osmt.config.INDEX_RICHSKILL_DOC import edu.wgu.osmt.config.QUOTED_SEARCH_REGEX_PATTERN import edu.wgu.osmt.db.PublishStatus import edu.wgu.osmt.elasticsearch.FindsAllByPublishStatus @@ -21,6 +22,7 @@ import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate import org.springframework.data.elasticsearch.core.SearchHits +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories @@ -46,6 +48,11 @@ interface CustomRichSkillQueries : FindsAllByPublishStatus { fun findSimilar(apiSimilaritySearch: ApiSimilaritySearch): SearchHits fun occupationQueries(query: String): NestedQueryBuilder + + + fun deleteIndex() { + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_RICHSKILL_DOC)).delete(); + } } class CustomRichSkillQueriesImpl @Autowired constructor(override val elasticSearchTemplate: ElasticsearchRestTemplate) : From 621ac0ee516e57846ebf6e173cda8120e3bbb10e Mon Sep 17 00:00:00 2001 From: John Kallies <3021949+JohnKallies@users.noreply.github.com> Date: Fri, 6 Jan 2023 10:38:06 -0500 Subject: [PATCH 2/2] clean up from review feedback --- .../kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt | 2 +- .../wgu/osmt/elasticsearch/ElasticSearchAdminController.kt | 7 ++++--- api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt | 2 +- api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt | 2 +- .../main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt index f8c343373..d46b4a155 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/collection/CollectionEsRepo.kt @@ -38,7 +38,7 @@ interface CustomCollectionQueries : FindsAllByPublishStatus { ): SearchHits fun deleteIndex() { - elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_COLLECTION_DOC)).delete(); + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_COLLECTION_DOC)).delete() } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt index 510160c9b..3d360cc04 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/elasticsearch/ElasticSearchAdminController.kt @@ -4,12 +4,13 @@ import edu.wgu.osmt.RoutePaths.ES_ADMIN_DELETE_INDICES import edu.wgu.osmt.RoutePaths.ES_ADMIN_REINDEX import edu.wgu.osmt.config.AppConfig import edu.wgu.osmt.security.OAuthHelper -import edu.wgu.osmt.task.* import org.springframework.beans.factory.annotation.Autowired -import org.springframework.http.* +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity import org.springframework.stereotype.Controller import org.springframework.transaction.annotation.Transactional -import org.springframework.web.bind.annotation.* +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.server.ResponseStatusException import java.util.concurrent.ForkJoinPool diff --git a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt index 59d422192..aab76bfa4 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/jobcode/JobCodeEsRepo.kt @@ -20,7 +20,7 @@ interface CustomJobCodeRepository { fun typeAheadSearch(query: String): SearchHits fun deleteIndex() { - elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_JOBCODE_DOC)).delete(); + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_JOBCODE_DOC)).delete() } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt index 5629828c0..48ac60879 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/keyword/KeywordEsRepo.kt @@ -18,7 +18,7 @@ interface CustomKeywordRepository { fun typeAheadSearch(query: String, type: KeywordTypeEnum): SearchHits fun deleteIndex() { - elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_KEYWORD_DOC)).delete(); + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_KEYWORD_DOC)).delete() } } diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt index 4dc8c38fc..f02df9d60 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillEsRepo.kt @@ -51,7 +51,7 @@ interface CustomRichSkillQueries : FindsAllByPublishStatus { fun deleteIndex() { - elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_RICHSKILL_DOC)).delete(); + elasticSearchTemplate.indexOps(IndexCoordinates.of(INDEX_RICHSKILL_DOC)).delete() } }