From ba65e016080dc37f1389625eb78ceb317785bc0f Mon Sep 17 00:00:00 2001 From: Edwin Santizo Date: Sun, 11 Dec 2022 13:54:25 -0700 Subject: [PATCH 1/2] Adding RSD to collection fix there was an error in adding RSDs to a collection in which some RSDs would not show up after adding them to a collection (even though pop up message shows added 'X' amount). Found that we were adding duplicates in which the system did not allow and discarded the duplicates. This was caused because of a sort issue in which user wouldn't notice they were adding a duplicate RSD to collection because sort would return inconsistent list of RSDs each time. Fixed this with by adding sort to name and not just category. New sort now returns consistent list but it would return with a case senstive list i.e capital letters came before lowercase which meant 'Academic' would come after "AWS". Fixed this by creating a normalizer for search results in which it lowercases results, sorted it and then returned results of search. --- .../edu/wgu/osmt/api/model/ApiSortEnum.kt | 8 ++++-- .../edu/wgu/osmt/richskill/RichSkillDoc.kt | 27 ++++++++++++------- .../resources/elasticsearch/settings.json | 7 +++++ 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/api/model/ApiSortEnum.kt b/api/src/main/kotlin/edu/wgu/osmt/api/model/ApiSortEnum.kt index 0a386ac10..06d360087 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/api/model/ApiSortEnum.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/api/model/ApiSortEnum.kt @@ -37,10 +37,14 @@ interface SortOrderCompanion where T: SortOrder{ */ enum class SkillSortEnum(override val apiValue: String) : SortOrder { CategoryAsc("name.asc") { - override val sort = Sort.by("category.keyword").ascending() + override val sort = Sort.by( + Sort.Order.asc("category.sort_insensitive"), + Sort.Order.asc("name.sort_insensitive")) }, CategoryDesc("name.desc") { - override val sort = Sort.by("category.keyword").descending() + override val sort = Sort.by( + Sort.Order.desc("category.sort_insensitive"), + Sort.Order.asc("name.sort_insensitive")) }, NameAsc("skill.asc") { override val sort = Sort.by(nameKeyword).ascending() diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt index 670119264..fd530b5df 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt @@ -44,7 +44,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonProperty("skillName") @@ -55,7 +56,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonProperty("skillStatement") @@ -67,7 +69,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonProperty @@ -79,7 +82,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonProperty("author") @@ -94,7 +98,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonProperty("keywords") @@ -109,7 +114,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonIgnore @@ -120,7 +126,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonIgnore @@ -131,7 +138,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonIgnore @@ -142,7 +150,8 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword) + InnerField(suffix = "keyword", type = Keyword), + InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") ] ) @get:JsonIgnore diff --git a/api/src/main/resources/elasticsearch/settings.json b/api/src/main/resources/elasticsearch/settings.json index 56c8235b1..76939bee5 100644 --- a/api/src/main/resources/elasticsearch/settings.json +++ b/api/src/main/resources/elasticsearch/settings.json @@ -13,6 +13,13 @@ "lowercase" ] } + }, + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "char_filter": [], + "filter": ["lowercase"] + } } } } From 2cbe005af2ffc1852d15dfac643566891c082a34 Mon Sep 17 00:00:00 2001 From: Edwin Santizo Date: Wed, 14 Dec 2022 10:49:19 -0700 Subject: [PATCH 2/2] removed normalizer from innerField for all except name and category --- .../edu/wgu/osmt/richskill/RichSkillDoc.kt | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt index fd530b5df..6fa2ce2ca 100644 --- a/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt +++ b/api/src/main/kotlin/edu/wgu/osmt/richskill/RichSkillDoc.kt @@ -56,8 +56,7 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword), - InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") + InnerField(suffix = "keyword", type = Keyword) ] ) @get:JsonProperty("skillStatement") @@ -82,8 +81,7 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword), - InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") + InnerField(suffix = "keyword", type = Keyword) ] ) @get:JsonProperty("author") @@ -98,8 +96,7 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword), - InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") + InnerField(suffix = "keyword", type = Keyword) ] ) @get:JsonProperty("keywords") @@ -114,8 +111,7 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword), - InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") + InnerField(suffix = "keyword", type = Keyword) ] ) @get:JsonIgnore @@ -126,8 +122,7 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword), - InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") + InnerField(suffix = "keyword", type = Keyword) ] ) @get:JsonIgnore @@ -138,8 +133,7 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword), - InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") + InnerField(suffix = "keyword", type = Keyword) ] ) @get:JsonIgnore @@ -150,8 +144,7 @@ data class RichSkillDoc( otherFields = [ InnerField(suffix = "", type = Search_As_You_Type), InnerField(suffix = "raw", analyzer = "whitespace_exact", type = Text), - InnerField(suffix = "keyword", type = Keyword), - InnerField(suffix = "sort_insensitive", type = Keyword, normalizer = "lowercase_normalizer") + InnerField(suffix = "keyword", type = Keyword) ] ) @get:JsonIgnore