Normalize website while creating organization (#1412)

Author: skwowet

backend/src/services/organizationService.ts

@@ -1,4 +1,5 @@
 import { LoggerBase } from '@crowd/logging'
+import { websiteNormalizer } from '@crowd/common'
 import { CLEARBIT_CONFIG, IS_TEST_ENV } from '../conf'
 import MemberRepository from '../database/repositories/memberRepository'
 import organizationCacheRepository from '../database/repositories/organizationCacheRepository'
@@ -43,6 +44,11 @@ export default class OrganizationService extends LoggerBase {
         transaction,
       })
 
+      // Normalize the website URL if it exists
+      if (data.website) {
+        data.website = websiteNormalizer(data.website)
+      }
+
       // if cache exists, merge current data with cache data
       // if it doesn't exist, create it from incoming data
       if (cache) {
@@ -146,6 +152,11 @@ export default class OrganizationService extends LoggerBase {
         })
       }
 
+      // Normalize the website URL if it exists
+      if (data.website) {
+        data.website = websiteNormalizer(data.website)
+      }
+
       const record = await OrganizationRepository.update(id, data, {
         ...this.options,
         transaction,

services/apps/data_sink_worker/src/service/organization.service.ts

@@ -6,6 +6,7 @@ import { OrganizationRepository } from '@/repo/organization.repo'
 import { DbStore } from '@crowd/database'
 import { Logger, LoggerBase, getChildLogger } from '@crowd/logging'
 import { IOrganization, IOrganizationSocial, PlatformType } from '@crowd/types'
+import { websiteNormalizer } from '@crowd/common'
 
 export class OrganizationService extends LoggerBase {
   private readonly repo: OrganizationRepository
@@ -23,6 +24,11 @@ export class OrganizationService extends LoggerBase {
   ): Promise<string> {
     data = this.normalizeSocialFields(data)
 
+    // Normalize the website URL if it exists
+    if (data.website) {
+      data.website = websiteNormalizer(data.website)
+    }
+
     // find from cache by name
     let cached = await this.repo.findCacheByName(data.name)
 

services/libs/common/src/index.ts

@@ -11,3 +11,4 @@ export * from './requestThrottler'
 export * from './rawQueryParser'
 export * from './byteLength'
 export * from './http'
+export * from './websiteNormalizer'

services/libs/common/src/websiteNormalizer.ts

@@ -0,0 +1,15 @@
+export const websiteNormalizer = (website: string): string => {
+  // Prepends https:// to make valid URL
+  const completeUrl = website.includes('://') ? website : 'https://' + website
+
+  const url = new URL(completeUrl)
+  const hostname = url.hostname
+
+  const parts = hostname.split('.')
+  // Ignore subdomains, return only domain and TLD
+  if (parts.length > 2) {
+    return parts.slice(-2).join('.')
+  }
+
+  return hostname
+}