⚙️ Enhance caching mechanism and add refresh option for secret retrieval

libraries/deno-commons-mod.ts

@@ -1,15 +1,23 @@
 // Reference:
 // - https://docs.deno.com/runtime/fundamentals/testing/
 
-import {decodeBase64, encodeBase64} from "jsr:@std/encoding/base64";
+import { decodeBase64, encodeBase64 } from "jsr:@std/encoding/base64";
-import {encodeBase32} from "jsr:@std/encoding/base32";
+import { encodeBase32 } from "jsr:@std/encoding/base32";
-import {dirname, fromFileUrl} from "jsr:@std/path";
+import { dirname, fromFileUrl } from "jsr:@std/path";
-import {toArrayBuffer} from "jsr:@std/streams";
+import { toArrayBuffer } from "jsr:@std/streams";
-import {spawn, SpawnOptionsWithoutStdio} from "node:child_process";
+import { spawn, SpawnOptionsWithoutStdio } from "node:child_process";
-import {createWriteStream, mkdir, readFile, readFileSync, rm, stat, writeFile,} from "node:fs";
+import {
-import {pipeline} from "node:stream";
+    createWriteStream,
-import {promisify} from "node:util";
+    mkdir,
-import {getRandomValues} from "node:crypto";
+    readFile,
+    readFileSync,
+    rm,
+    stat,
+    writeFile,
+} from "node:fs";
+import { pipeline } from "node:stream";
+import { promisify } from "node:util";
+import { getRandomValues } from "node:crypto";
 
 // reference: https://docs.deno.com/examples/hex_base64_encoding/
 // import { decodeBase64, encodeBase64 } from "jsr:@std/encoding/base64";
@@ -709,7 +717,8 @@ class Logger {
     __debug: boolean = false;
 
     constructor() {
-        this.__debug = osEnv("LOGGER") === "*";
+        this.__debug = osEnv("LOGGER") === "*" ||
+            osEnv("DENO_COMMONS_LOGGER") === "*";
     }
 
     // deno-lint-ignore no-explicit-any
@@ -1080,12 +1089,11 @@ export async function decryptDefault(
     const key = await __getMasterEncryptionKey();
     const nonce = input.slice(0, 12);
     const ciphertextWithTag = input.slice(12);
-    const plaintext = await crypto.subtle.decrypt(
+    return await crypto.subtle.decrypt(
         { name: "AES-GCM", iv: nonce },
         key,
         ciphertextWithTag,
     );
-    return plaintext;
 }
 
 export async function decryptDefaultAsString(
@@ -1095,6 +1103,94 @@ export async function decryptDefaultAsString(
     return new TextDecoder().decode(plaintext);
 }
 
+interface LocalCacheOptions {
+    basePath?: string;
+    tryRefresh?: boolean;
+    defaultCacheTimeSecs?: number;
+    maxCacheTimeSecs?: number;
+}
+
+interface LocalCacheValue<T> {
+    value: T;
+    cacheTimestampSecs: number;
+}
+
+export async function readWithLocalCache<T>(
+    cacheKey: string,
+    read: () => Promise<T>,
+    options?: LocalCacheOptions,
+): Promise<T> {
+    if (!cacheKey || !read) {
+        throw new Error("Key and read callback function are both required.");
+    }
+    const cacheKeyHash = await sha256AndBase32OfString(cacheKey);
+    const basePath = options?.basePath ?? "~/.cache/local-cache";
+    const cacheDir = joinPath(
+        basePath,
+        cacheKeyHash.substring(0, 2),
+        cacheKeyHash.substring(2, 4),
+    );
+    if (!await existsPath(cacheDir)) {
+        await makeDirectory(cacheDir, true);
+    }
+    const cacheFile = joinPath(cacheDir, cacheKeyHash);
+    log.debug(`key: ${cacheKey} --> ${cacheFile}`);
+    const cachedContent = await readFileToString(cacheFile);
+
+    let cachedValue: LocalCacheValue<T> | null = null;
+    if (cachedContent) {
+        try {
+            cachedValue = JSON.parse(
+                await decryptDefaultAsString(cachedContent),
+            ) as LocalCacheValue<T>;
+            log.debug(`key: ${cacheKey}, cached value:`, cachedValue);
+        } catch (e) {
+            log.debug(`key :${cacheKey}, decrypt error`, e);
+        }
+    }
+    const nowSecs = Math.floor(Date.now() / 1000);
+    if (cachedValue && options.defaultCacheTimeSecs && !options?.tryRefresh) {
+        if (
+            (nowSecs - cachedValue.cacheTimestampSecs) <
+                options.defaultCacheTimeSecs
+        ) {
+            log.debug(
+                `key: ${cacheKey} cache valid, now: ${nowSecs}, cacheTimestampSecs: ${cachedValue.cacheTimestampSecs}, default cache time secs: ${options.defaultCacheTimeSecs}`,
+            );
+            return cachedValue.value;
+        }
+    }
+    let lastError = null;
+    try {
+        const value = await read();
+        await writeStringToFile(
+            cacheFile,
+            await encryptDefaultAsBase64(
+                JSON.stringify({
+                    value: value,
+                    cacheTimestampSecs: nowSecs,
+                } as LocalCacheValue<T>),
+            ),
+        );
+        return value;
+    } catch (e) {
+        lastError = e;
+        log.debug("error in read", e);
+    }
+    if (cachedValue && options.maxCacheTimeSecs) {
+        if (
+            (nowSecs - cachedValue.cacheTimestampSecs) <
+                options.maxCacheTimeSecs
+        ) {
+            log.debug(
+                `key: ${cacheKey} fallback cached value, now: ${nowSecs}, cacheTimestampSecs: ${cachedValue.cacheTimestampSecs}, max cache time secs: ${options.maxCacheTimeSecs}`,
+            );
+            return cachedValue.value;
+        }
+    }
+    throw lastError;
+}
+
 export async function getKeyRingPassword(
     service: string,
     user: string,
@@ -1535,6 +1631,8 @@ export type SecretValueRunEnv = "ALIBABA_CLOUD" | "HATTER_CLI";
 
 export interface GetSecretValueOptions {
     runEnv?: SecretValueRunEnv;
+    localCache?: boolean;
+    tryRefresh?: boolean;
 }
 
 export function getRunEnv(): stirng | null {
@@ -1555,11 +1653,10 @@ function toGetSecretValueOptions(
     return runEnvOrOptions as GetSecretValueOptions;
 }
 
-export async function getSecretValue(
+async function __getSecretValue(
     key: string,
-    runEnvOrOptions?: SecretValueRunEnv | GetSecretValueOptions,
+    options?: GetSecretValueOptions,
 ): Promise<string> {
-    const options = toGetSecretValueOptions(runEnvOrOptions);
     const runEnv = options?.runEnv ?? getRunEnv();
     if (runEnv == "ALIBABA_CLOUD") {
         return await getSecretValueViaAlibabaCloudInstanceIdentity(key);
@@ -1567,6 +1664,24 @@ export async function getSecretValue(
     return await getSecretValueViaHatterCli(key);
 }
 
+export async function getSecretValue(
+    key: string,
+    runEnvOrOptions?: SecretValueRunEnv | GetSecretValueOptions,
+): Promise<string> {
+    const options = toGetSecretValueOptions(runEnvOrOptions);
+    return await readWithLocalCache(
+        `secret_value:${key}`,
+        async (): Promise<string> => {
+            return await __getSecretValue(key, options);
+        },
+        {
+            tryRefresh: options.tryRefresh,
+            defaultCacheTimeSecs: 3600,
+            maxCacheTimeSecs: 3600 * 24,
+        },
+    );
+}
+
 export interface StsToken {
     mode: string;
     expiration: string;

single-scripts/get-secret.ts

@@ -1,10 +1,10 @@
 #!/usr/bin/env -S deno run -A
 
-import {args, exit, getSecretValue,} from "https://script.hatter.ink/@62/deno-commons-mod.ts";
+import {args, exit, getSecretValue,} from "https://script.hatter.ink/@66/deno-commons-mod.ts";
 import {parseArgs} from "jsr:@std/cli/parse-args";
 
 const flags = parseArgs(args(), {
-    boolean: ["help"],
+    boolean: ["help", "refresh"],
     string: ["id"],
 });
 
@@ -19,7 +19,11 @@ if (flags.help) {
 }
 
 if (flags.id) {
-    console.log(await getSecretValue(flags.id));
+    console.log(
+        await getSecretValue(flags.id, {
+            tryRefresh: flags.refresh,
+        }),
+    );
     exit(0);
 } else {
     console.error("get-secret.ts --id is required");