ENSRainbow client graceful retry mechanism

.changeset/purple-clubs-strive.md

@@ -0,0 +1,5 @@
+---
+"ensindexer": patch
+---
+
+Add retry-with-backoff for ENSRainbow `heal()` calls during indexing. Transient failures (network errors and server errors) are retried up to 3 times with exponential backoff, with a warning logged on each failed attempt. This prevents a single transient ENSRainbow error from causing indexing to fail.

apps/ensindexer/src/lib/ensraibow-api-client.ts

@@ -1,11 +1,8 @@
 import config from "@/config";
 
-import { EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk";
+import { type EnsRainbow, EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk";
 
-/**
- * Get a {@link EnsRainbowApiClient} instance.
- */
-export function getENSRainbowApiClient() {
+export function getENSRainbowApiClient(): EnsRainbow.ApiClient {
   const ensRainbowApiClient = new EnsRainbowApiClient({
     endpointUrl: config.ensRainbowUrl,
     labelSet: config.labelSet,

apps/ensindexer/src/lib/graphnode-helpers.test.ts

@@ -1,11 +1,22 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 import type { LabelHash } from "@ensnode/ensnode-sdk";
 
 import { setupConfigMock } from "@/lib/__test__/mockConfig";
 
 setupConfigMock(); // setup config mock before importing dependent modules
 
+// Use real p-retry logic but with 0 timeouts so tests don't incur actual backoff delays.
+vi.mock("p-retry", async () => {
+  const { default: actualPRetry } = await vi.importActual<typeof import("p-retry")>("p-retry");
+  return {
+    default: (
+      fn: Parameters<typeof actualPRetry>[0],
+      options?: Parameters<typeof actualPRetry>[1],
+    ) => actualPRetry(fn, { ...options, minTimeout: 0, maxTimeout: 0 }),
+  };
+});
+
 // Mock fetch globally to prevent real network calls
 global.fetch = vi.fn();
 
@@ -132,4 +143,130 @@ describe("labelByLabelHash", () => {
     ).rejects.toThrow(/Invalid labelHash length/i);
     expect(fetch).not.toHaveBeenCalled();
   });
+
+  describe("retry behavior", () => {
+    afterEach(() => {
+      vi.restoreAllMocks();
+    });
+
+    // Use unique labelHashes in each test to prevent LRU cache hits from other tests
+    // carrying over cacheable responses (HealSuccess, HealNotFoundError) and bypassing fetch.
+
+    it("retries on network/fetch failure and succeeds on a later attempt", async () => {
+      const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+      (fetch as any)
+        .mockRejectedValueOnce(new Error("network error"))
+        .mockRejectedValueOnce(new Error("network error"))
+        .mockResolvedValue({
+          ok: true,
+          json: () => Promise.resolve({ status: "success", label: "nick" }),
+        });
+
+      const result = await labelByLabelHash(
+        "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" as LabelHash,
+      );
+
+      expect(result).toEqual("nick");
+      expect(fetch).toHaveBeenCalledTimes(3);
+      expect(warnSpy).toHaveBeenCalledTimes(2);
+    });
+
+    it("retries on HealServerError and succeeds on a later attempt", async () => {
+      const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+      (fetch as any)
+        .mockResolvedValueOnce({
+          ok: true,
+          json: () =>
+            Promise.resolve({ status: "error", error: "Internal server error", errorCode: 500 }),
+        })
+        .mockResolvedValue({
+          ok: true,
+          json: () => Promise.resolve({ status: "success", label: "vitalik" }),
+        });
+
+      const result = await labelByLabelHash(
+        "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" as LabelHash,
+      );
+
+      expect(result).toEqual("vitalik");
+      expect(fetch).toHaveBeenCalledTimes(2);
+      expect(warnSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it("does not retry HealNotFoundError — returns null after a single call", async () => {
+      (fetch as any).mockResolvedValue({
+        ok: true,
+        json: () => Promise.resolve({ status: "error", error: "Label not found", errorCode: 404 }),
+      });
+
+      const result = await labelByLabelHash(
+        "0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" as LabelHash,
+      );
+
+      expect(result).toBeNull();
+      expect(fetch).toHaveBeenCalledTimes(1);
+    });
+
+    it("does not retry HealBadRequestError — throws after a single call", async () => {
+      (fetch as any).mockResolvedValue({
+        ok: true,
+        json: () =>
+          Promise.resolve({
+            status: "error",
+            error: "Invalid labelhash",
+            errorCode: 400,
+          }),
+      });
+
+      await expect(
+        labelByLabelHash(
+          "0xdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" as LabelHash,
+        ),
+      ).rejects.toThrow(/Invalid labelhash/i);
+      expect(fetch).toHaveBeenCalledTimes(1);
+    });
+
+    it("throws after exhausting retries on persistent network/fetch failures", async () => {
+      const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+      (fetch as any).mockRejectedValue(new Error("network error"));
+
+      await expect(
+        labelByLabelHash(
+          "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" as LabelHash,
+        ),
+      ).rejects.toThrow(/ENSRainbow Heal Request Failed/i);
+
+      // 1 initial attempt + 3 retries = 4 total
+      expect(fetch).toHaveBeenCalledTimes(4);
+      expect(warnSpy).toHaveBeenCalledTimes(4);
+    });
+
+    it("throws after exhausting retries on persistent HealServerError responses", async () => {
+      const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+      (fetch as any).mockResolvedValue({
+        ok: true,
+        json: () =>
+          Promise.resolve({ status: "error", error: "Internal server error", errorCode: 500 }),
+      });
+
+      const err = await labelByLabelHash(
+        "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" as LabelHash,
+      ).then(
+        () => null as unknown as Error,
+        (e: unknown) => e as Error,
+      );
+      expect(err).not.toBeNull();
+      expect(err!.message).toMatch(/Internal server error/i);
+      expect(err!.cause).toBeInstanceOf(Error);
+      expect((err!.cause as Error).message).toBe("Internal server error");
+
+      // 1 initial attempt + 3 retries = 4 total
+      expect(fetch).toHaveBeenCalledTimes(4);
+      expect(warnSpy).toHaveBeenCalledTimes(4);
+    });
+  });
 });

apps/ensindexer/src/lib/graphnode-helpers.ts

@@ -1,5 +1,7 @@
+import pRetry from "p-retry";
+
 import type { LabelHash, LiteralLabel } from "@ensnode/ensnode-sdk";
-import { ErrorCode, isHealError } from "@ensnode/ensrainbow-sdk";
+import { type EnsRainbow, ErrorCode, isHealError } from "@ensnode/ensrainbow-sdk";
 
 import { getENSRainbowApiClient } from "@/lib/ensraibow-api-client";
 
@@ -10,14 +12,74 @@ const ensRainbowApiClient = getENSRainbowApiClient();
  * It mirrors the `ens.nameByHash` function implemented in GraphNode:
  * https://github.com/graphprotocol/graph-node/blob/3c448de/runtime/test/wasm_test/api_version_0_0_4/ens_name_by_hash.ts#L9-L11
  *
+ * ## Transient vs. non-transient errors
+ *
+ * ENSIndexer calls this function for every indexed ENS event that requires label healing. A single
+ * transient failure (e.g. a momentary network blip or a brief ENSRainbow server hiccup) would
+ * otherwise crash the ENSIndexer process, forcing a full restart and re-index from the last
+ * checkpoint. To avoid this disproportionate impact, transient failures are retried with
+ * exponential backoff (up to 3 retries, with backoff delays between 1 and 30 seconds between attempts) before the error is thrown:
+ * - Network/fetch failure: heal() throws because the ENSRainbow service was unreachable.
+ * - HealServerError (errorCode 500): ENSRainbow returned a transient server-side error.
+ *
+ * Non-transient outcomes are thrown immediately without retry, because retrying would not change
+ * the result:
+ * - HealSuccess: the label was healed successfully; returned.
+ * - HealNotFoundError (errorCode 404): no label is known for this labelHash; null returned.
+ * - HealBadRequestError (errorCode 400): the labelHash is malformed; retrying would not help.
+ *
+ * ## Non-recoverable throws
+ *
+ * Any throw from this function is not recoverable or has exceeded the max retries. It propagates to the calling indexing handler
+ * (Registry, Registrar, ThreeDNSToken, label-db-helpers) which does not catch it, causing the
+ * ENSIndexer process to terminate.
+ *
  * @returns the original label if found, or null if not found for the labelHash.
- * @throws if the labelHash is not correctly formatted, or server error occurs, or connection error occurs.
- **/
+ * @throws if ENSRainbow returns a non-retryable error response (e.g. HealBadRequestError / 400),
+ *   or if a transient error (network failure, HealServerError / 500) persists after all retries
+ *   are exhausted.
+ */
 export async function labelByLabelHash(labelHash: LabelHash): Promise<LiteralLabel | null> {
+  // Reset at the start of each attempt so that after p-retry exhaustion we can distinguish
+  // "last failure was HealServerError" (set) from "last failure was a network throw" (undefined).
+  let lastServerError: EnsRainbow.HealServerError | undefined;
+
   let response: Awaited<ReturnType<typeof ensRainbowApiClient.heal>>;
+
   try {
-    response = await ensRainbowApiClient.heal(labelHash);
+    response = await pRetry(
+      async () => {
+        lastServerError = undefined;
+        const result = await ensRainbowApiClient.heal(labelHash);
+
+        if (isHealError(result) && result.errorCode === ErrorCode.ServerError) {
+          lastServerError = result;
+          throw new Error(result.error);
+        }
+
+        return result;
+      },
+      {
+        retries: 3,
+        minTimeout: 1_000,
+        maxTimeout: 30_000,
+        onFailedAttempt({ error, attemptNumber, retriesLeft }) {
+          console.warn(
+            `ENSRainbow heal failed (attempt ${attemptNumber}): ${error.message}. ${retriesLeft} retries left.`,
+          );
+        },
+      },
+    );
   } catch (error) {
+    if (lastServerError) {
+      // Not recoverable; causes the ENSIndexer process to terminate.
+      throw new Error(
+        `Error healing labelHash: "${labelHash}". Error (${lastServerError.errorCode}): ${lastServerError.error}.`,
+        { cause: error },
+      );
+    }
+
+    // Not recoverable; causes the ENSIndexer process to terminate.
     if (error instanceof Error) {
       error.message = `ENSRainbow Heal Request Failed: ENSRainbow unavailable at '${ensRainbowApiClient.getOptions().endpointUrl}'.`;
     }
@@ -26,14 +88,13 @@ export async function labelByLabelHash(labelHash: LabelHash): Promise<LiteralLab
   }
 
   if (isHealError(response)) {
-    // no original label found for the labelHash
     if (response.errorCode === ErrorCode.NotFound) return null;
 
+    // Not recoverable; causes the ENSIndexer process to terminate.
     throw new Error(
       `Error healing labelHash: "${labelHash}". Error (${response.errorCode}): ${response.error}.`,
     );
   }
 
-  // original label found for the labelHash
   return response.label as LiteralLabel;
 }

apps/ensindexer/src/lib/public-config-builder/public-config-builder.test.ts

@@ -7,7 +7,7 @@ import {
   type EnsRainbowPublicConfig,
   PluginName,
 } from "@ensnode/ensnode-sdk";
-import type { EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk";
+import type { EnsRainbow } from "@ensnode/ensrainbow-sdk";
 
 import { PublicConfigBuilder } from "./public-config-builder";
 
@@ -98,7 +98,7 @@ describe("PublicConfigBuilder", () => {
       // Arrange
       const ensRainbowClientMock = {
         config: vi.fn().mockResolvedValue(mockEnsRainbowConfig),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       setupStandardMocks();
       const mockPublicConfig = createMockPublicConfig();
@@ -142,7 +142,7 @@ describe("PublicConfigBuilder", () => {
       // Arrange
       const ensRainbowClientMock = {
         config: vi.fn().mockResolvedValue(mockEnsRainbowConfig),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       setupStandardMocks();
       const mockPublicConfig = createMockPublicConfig();
@@ -180,7 +180,7 @@ describe("PublicConfigBuilder", () => {
 
       const ensRainbowClientMock = {
         config: vi.fn().mockResolvedValue(mockEnsRainbowConfig),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       vi.mocked(getEnsIndexerVersion).mockReturnValue("2.0.0");
       vi.mocked(getNodeJsVersion).mockReturnValue("22.0.0");
@@ -222,7 +222,7 @@ describe("PublicConfigBuilder", () => {
 
       const ensRainbowClientMock = {
         config: vi.fn().mockResolvedValue(customEnsRainbowConfig),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       setupStandardMocks();
       vi.mocked(validateEnsIndexerPublicConfig).mockReturnValue(customConfig);
@@ -244,7 +244,7 @@ describe("PublicConfigBuilder", () => {
       const ensRainbowError = new Error("ENSRainbow service unavailable");
       const ensRainbowClientMock = {
         config: vi.fn().mockRejectedValue(ensRainbowError),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       setupStandardMocks();
 
@@ -260,7 +260,7 @@ describe("PublicConfigBuilder", () => {
       // Arrange
       const ensRainbowClientMock = {
         config: vi.fn().mockResolvedValue(mockEnsRainbowConfig),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       setupStandardMocks();
 
@@ -281,7 +281,7 @@ describe("PublicConfigBuilder", () => {
       // Arrange
       const ensRainbowClientMock = {
         config: vi.fn().mockResolvedValue(mockEnsRainbowConfig),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       setupStandardMocks();
 
@@ -310,7 +310,7 @@ describe("PublicConfigBuilder", () => {
           callCount++;
           return Promise.resolve(mockEnsRainbowConfig);
         }),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       setupStandardMocks();
 
@@ -339,7 +339,7 @@ describe("PublicConfigBuilder", () => {
       // Arrange
       const ensRainbowClientMock = {
         config: vi.fn().mockRejectedValueOnce(new Error("ENSRainbow down")),
-      } as unknown as EnsRainbowApiClient;
+      } as unknown as EnsRainbow.ApiClient;
 
       const builder = new PublicConfigBuilder(ensRainbowClientMock);
 

apps/ensindexer/src/lib/public-config-builder/public-config-builder.ts

@@ -6,7 +6,7 @@ import {
   validateEnsIndexerPublicConfig,
   validateEnsIndexerVersionInfo,
 } from "@ensnode/ensnode-sdk";
-import type { EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk";
+import type { EnsRainbow } from "@ensnode/ensrainbow-sdk";
 
 import { getEnsIndexerVersion, getNodeJsVersion, getPackageVersion } from "@/lib/version-info";
 
@@ -17,7 +17,7 @@ export class PublicConfigBuilder {
    * Used to fetch ENSRainbow Public Config, which is part of
    * the ENSIndexer Public Config.
    */
-  private ensRainbowClient: EnsRainbowApiClient;
+  private ensRainbowClient: EnsRainbow.ApiClient;
 
   /**
    * Immutable ENSIndexer Public Config
@@ -30,7 +30,7 @@ export class PublicConfigBuilder {
   /**
    * @param ensRainbowClient ENSRainbow Client instance used to fetch ENSRainbow Public Config
    */
-  constructor(ensRainbowClient: EnsRainbowApiClient) {
+  constructor(ensRainbowClient: EnsRainbow.ApiClient) {
     this.ensRainbowClient = ensRainbowClient;
   }