updats

libraries/deno-commons-mod.ts

@@ -1,11 +1,13 @@
 // Reference:
 // - https://docs.deno.com/runtime/fundamentals/testing/
 
-import { decodeBase64, encodeBase64 } from "jsr:@std/encoding/base64";
-import { dirname, fromFileUrl } from "jsr:@std/path";
-import { toArrayBuffer } from "jsr:@std/streams";
-import { spawn, SpawnOptionsWithoutStdio } from "node:child_process";
-import { mkdir, readFile, readFileSync, rm, writeFile } from "node:fs";
+import {decodeBase64, encodeBase64} from "jsr:@std/encoding/base64";
+import {dirname, fromFileUrl} from "jsr:@std/path";
+import {toArrayBuffer} from "jsr:@std/streams";
+import {spawn, SpawnOptionsWithoutStdio} from "node:child_process";
+import {createWriteStream, mkdir, readFile, readFileSync, rm, writeFile,} from "node:fs";
+import {pipeline} from "node:stream";
+import {promisify} from "node:util";
 
 // reference: https://docs.deno.com/examples/hex_base64_encoding/
 // import { decodeBase64, encodeBase64 } from "jsr:@std/encoding/base64";
@@ -930,7 +932,7 @@ export async function makeDirectory(
     recursive?: boolean,
 ): Promise<void> {
     if (isDeno()) {
-        await Deno.mkdir(parentDirname, { recursive: recursive ?? true });
+        await Deno.mkdir(directory, { recursive: recursive ?? true });
     } else {
         return new Promise((resolve, reject) => {
             mkdir(directory, { recursive: recursive ?? true }, (err) => {
@@ -1138,3 +1140,112 @@ export function stringifySorted<T extends Record<string, any>>(
 export function stringifyPretty(object: any): string {
     return JSON.stringify(object, null, 2);
 }
+
+export async function sha256AndHexOfString(input: string): Promise<string> {
+    return uint8ArrayToHexString(new Uint8Array(await sha256OfString(input)));
+}
+
+export async function sha256OfString(input: string): Promise<ArrayBuffer> {
+    const data = new TextEncoder().encode(input);
+    return await crypto.subtle.digest("SHA-256", data);
+}
+
+export const BASE_FILE_CACHE_DIR = "~/.cache/commons-cache";
+
+export interface FetchFileWithCacheMeta {
+    url: string;
+    tag?: string;
+    cache_full_path: string;
+    download_time: number;
+}
+
+export interface FetchFileWithCacheOptions {
+    baseDir?: string;
+    tag?: string;
+    timeoutMillis?: number;
+    check_cache_file?: (
+        meta: FetchFileWithCacheMeta,
+    ) => Promise<"valid" | "try_update" | "invalid">;
+    after_cache_file?: (meta: FetchFileWithCacheMeta) => Promise<void>;
+}
+
+export function getFilenameFromUrl(url: string): string {
+    const array = new URL(url).pathname.split("/");
+    let filename = array[array.length - 1].trim();
+    if (!filename) {
+        return "unnamed";
+    }
+    try {
+        return decodeURIComponent(filename);
+    } catch (e) {
+        return filename;
+    }
+}
+
+export async function fetchFileWithCache(
+    url: string,
+    options?: FetchFileWithCacheOptions,
+): Promise<FetchFileWithCacheMeta> {
+    const urlSha256 = await sha256AndHexOfString(url);
+    const fileCacheDir = joinPath(
+        resolveFilename(options?.baseDir ?? BASE_FILE_CACHE_DIR),
+        urlSha256,
+    );
+    const fileCacheMetaFile = fileCacheDir + ".meta";
+    const fileCacheFile = joinPath(fileCacheDir, getFilenameFromUrl(url));
+
+    let cachedMeta = null;
+    const fileCacheMetaContent = await readFileToString(fileCacheMetaFile);
+    if (fileCacheMetaContent) {
+        const meta = JSON.parse(fileCacheMetaContent) as FetchFileWithCacheMeta;
+        if (options?.check_cache_file) {
+            const checkCacheFileResult = await options.check_cache_file(meta);
+            if (checkCacheFileResult == "valid") {
+                return meta;
+            } else if (checkCacheFileResult == "try_update") {
+                cachedMeta = meta;
+            } else {
+                // invalid meta
+            }
+        } else {
+            return meta;
+        }
+    }
+
+    if (!await existsPath(fileCacheDir)) {
+        await makeDirectory(fileCacheDir);
+    }
+
+    try {
+        const fetchResponse = await fetchDataWithTimeout(url, {
+            timeoutMillis: options?.timeoutMillis ?? 10 * 60 * 1000,
+        });
+        if (fetchResponse.status != 200) {
+            // invalid resource
+            throw new Error(
+                `Fetch ${url} failed, status: ${fetchResponse.status}`,
+            );
+        }
+        const writeFile = promisify(pipeline);
+        await writeFile(fetchResponse.body, createWriteStream(fileCacheFile));
+        const newCachedMeta = {
+            url: url,
+            tag: options?.tag,
+            cache_full_path: fileCacheFile,
+            download_time: Date.now(),
+        };
+        await writeStringToFile(
+            fileCacheMetaFile,
+            stringifyPretty(newCachedMeta),
+        );
+        if (options?.after_cache_file) {
+            await options.after_cache_file(newCachedMeta);
+        }
+        return newCachedMeta;
+    } catch (e) {
+        if (cachedMeta != null) {
+            return cachedMeta;
+        }
+        throw e;
+    }
+}