feat(archive): separate GitHub mirror upload into dedicated command

.github/workflows/main.yml

@@ -4,7 +4,7 @@ permissions:
   checks: write
   contents: write
 on:
-  # Scheduled trigger is disabled by default
+  # Scheduled trigger is disabled by default (use cron-job)
   # Uncommenting the following two lines will enable it
   # schedule:
   #   - cron: '0 16 * * *'
@@ -32,20 +32,27 @@ jobs:
           max_attempts: 5
           timeout_minutes: 10
           command: bun run src/main.ts archive
-      - name: Format output data
+      - name: Format output folder
         run: bun x oxfmt output
-      # - name: Git staging, commit, push
-      #   continue-on-error: true
-      #   run: |
-      #     git config user.name "github-actions[bot]"
-      #     git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-      #     git add output
-      #     git commit -m '[Auto] API update'
-      #     git push
       - name: Git commit and push
         uses: iarekylew00t/verified-bot-commit@v2
         with:
           message: "[Auto] API update"
           files: |
-            output
+            output/akEndfield
+            output/raw
+            output/mirror_file_list.json
+          if-no-commit: info
+      - name: Run GitHub mirror upload
+        run: bun run src/main.ts ghMirrorUpload
+        continue-on-error: true
+      - name: Format output folder
+        run: bun x oxfmt output
+      - name: Git commit and push
+        uses: iarekylew00t/verified-bot-commit@v2
+        with:
+          message: "[Auto] API update"
+          files: |
+            output/mirror_file_list.json
+            output/mirror_file_list_pending.json
           if-no-commit: info

output/mirror_file_list.json

@@ -974,4 +974,4 @@
     "mirror": "https://github.com/AetherArchive/beyond-hg-archive/releases/download/tag/1.0.14_bJBg3b40frDq9bOB_patches_Beyond_Release_v1d0-Rel-cn-5157154-10_prod_obt_bilibili_1_0_13.zip",
     "origStatus": false
   }
-]
+]

output/mirror_file_list_pending.json

@@ -0,0 +1 @@
+[]

src/cmd.ts

@@ -42,6 +42,22 @@ async function parseCommand() {
       },
       wrapHandler(cmds.archive),
     )
+    .command(
+      ['ghMirrorUpload'],
+      'Upload pending large binary file to GitHub mirror',
+      (yargs) => {
+        yargs.options({
+          'output-dir': {
+            alias: ['o'],
+            desc: 'Output root directory',
+            default: path.resolve('output'),
+            normalize: true,
+            type: 'string',
+          },
+        });
+      },
+      wrapHandler(cmds.ghMirrorUpload),
+    )
     .command(
       ['authTest [token] [email] [password]'],
       'Auth and gacha fetch test command',

src/cmds.ts

@@ -1,7 +1,9 @@
 import archive from './cmds/archive.js';
 import authTest from './cmds/authTest.js';
+import ghMirrorUpload from './cmds/ghMirrorUpload.js';
 
 export default {
   authTest,
   archive,
+  ghMirrorUpload,
 };

src/cmds/archive.ts

@@ -1,14 +1,11 @@
 import path from 'node:path';
-import { Octokit } from '@octokit/rest';
 import ky, { HTTPError } from 'ky';
 import { DateTime } from 'luxon';
 import PQueue from 'p-queue';
 import semver from 'semver';
-import YAML from 'yaml';
 import apiUtils from '../utils/api/index.js';
 import argvUtils from '../utils/argv.js';
 import appConfig from '../utils/config.js';
-import githubUtils from '../utils/github.js';
 import logger from '../utils/logger.js';
 import mathUtils from '../utils/math.js';
 import stringUtils from '../utils/string.js';
@@ -23,6 +20,12 @@ interface StoredData<T> {
   updatedAt: string;
 }
 
+interface MirrorFileEntry {
+  orig: string;
+  mirror: string;
+  origStatus: boolean;
+}
+
 interface GameTarget {
   name: string;
   region: 'os' | 'cn';
@@ -41,20 +44,12 @@ interface LauncherTarget {
   channel: number;
 }
 
-interface MirrorFileEntry {
-  orig: string;
-  mirror: string;
-  origStatus: boolean;
-}
-
 interface AssetToMirror {
   url: string;
   name: string | null;
 }
 
 // Global/Shared State
-let githubAuthCfg: any = null;
-let octoClient: Octokit | null = null;
 const assetsToMirror: AssetToMirror[] = [];
 const networkQueue = new PQueue({ concurrency: appConfig.threadCount.network });
 
@@ -572,64 +567,7 @@ async function fetchAndSaveLatestWebApis(gameTargets: GameTarget[]) {
   await networkQueue.onIdle();
 }
 
-// Mirroring and Cleanup
-async function checkMirrorFileDbStatus() {
-  logger.info('Checking mirrored files availability ...');
-  const dbPath = path.join(argvUtils.getArgv()['outputDir'], 'mirror_file_list.json');
-  const db = (await Bun.file(dbPath).json()) as MirrorFileEntry[];
-
-  for (const entry of db) {
-    networkQueue.add(async () => {
-      try {
-        await ky.head(entry.orig, {
-          headers: { 'User-Agent': appConfig.network.userAgent.minimum },
-          timeout: appConfig.network.timeout,
-          retry: { limit: appConfig.network.retryCount },
-        });
-        entry.origStatus = true;
-      } catch {
-        if (entry.origStatus) logger.trace(`Orig inaccessible: ${entry.orig.split('/').pop()}`);
-        entry.origStatus = false;
-      }
-    });
-  }
-  await networkQueue.onIdle();
-  await Bun.write(dbPath, JSON.stringify(db));
-}
-
-async function processMirrorQueue() {
-  const dbPath = path.join(argvUtils.getArgv()['outputDir'], 'mirror_file_list.json');
-  const db = (await Bun.file(dbPath).json()) as MirrorFileEntry[];
-
-  for (const { url, name } of assetsToMirror) {
-    const origUrl = stringUtils.removeQueryStr(url);
-    if (!db.find((e) => e.orig.includes(origUrl))) {
-      await githubUtils.uploadAsset(octoClient, githubAuthCfg, url, name);
-      if (githubAuthCfg) {
-        db.push({
-          orig: origUrl,
-          mirror: `https://github.com/${githubAuthCfg.github.relArchive.owner}/${githubAuthCfg.github.relArchive.repo}/releases/download/${githubAuthCfg.github.relArchive.tag}/${name ?? new URL(url).pathname.split('/').pop() ?? ''}`,
-          origStatus: true,
-        });
-        await Bun.write(dbPath, JSON.stringify(db));
-      }
-    }
-  }
-}
-
 async function mainCmdHandler() {
-  const authPath = 'config/config_auth.yaml';
-  if (await Bun.file(authPath).exists()) {
-    githubAuthCfg = YAML.parse(await Bun.file(authPath).text());
-    logger.info('Logging in to GitHub');
-    octoClient = new Octokit({ auth: githubAuthCfg.github.relArchive.token });
-  }
-
-  if (await githubUtils.checkIsActionRunning(githubAuthCfg)) {
-    logger.error('Duplicate execution detected');
-    return;
-  }
-
   const cfg = appConfig.network.api.akEndfield;
   const gameTargets: GameTarget[] = [
     {
@@ -701,12 +639,35 @@ async function mainCmdHandler() {
   await fetchAndSaveLatestLauncher(launcherTargets);
   await fetchAndSaveAllGameResRawData(gameTargets);
 
-  await checkMirrorFileDbStatus();
+  // Save pending assets to mirror
-  await processMirrorQueue();
+  const outputDir = argvUtils.getArgv()['outputDir'];
+  const pendingPath = path.join(outputDir, 'mirror_file_list_pending.json');
+  const dbPath = path.join(outputDir, 'mirror_file_list.json');
 
-  const relInfo = await githubUtils.getReleaseInfo(octoClient, githubAuthCfg);
+  let pendingData: AssetToMirror[] = [];
-  if (relInfo) {
+  if (await Bun.file(pendingPath).exists()) {
-    logger.info(`GitHub Releases total size: ${formatBytes(mathUtils.arrayTotal(relInfo.assets.map((a) => a.size)))}`);
+    pendingData = await Bun.file(pendingPath).json();
+  }
+  const db: MirrorFileEntry[] = (await Bun.file(dbPath).exists()) ? await Bun.file(dbPath).json() : [];
+  let addedCount = 0;
+
+  const uniqueAssetsToMirror = assetsToMirror.filter(
+    (asset, index, self) => index === self.findIndex((t) => t.url === asset.url),
+  );
+
+  for (const asset of uniqueAssetsToMirror) {
+    const origUrl = stringUtils.removeQueryStr(asset.url);
+    const dbExists = db.some((e) => e.orig.includes(origUrl));
+    const pendingExists = pendingData.some((e) => stringUtils.removeQueryStr(e.url) === origUrl);
+    if (!dbExists && !pendingExists) {
+      pendingData.push(asset);
+      addedCount++;
+    }
+  }
+
+  if (addedCount > 0) {
+    logger.info(`Saved ${addedCount} new assets to mirror pending list`);
+    await Bun.write(pendingPath, JSON.stringify(pendingData, null, 2));
   }
 }
 

src/cmds/ghMirrorUpload.ts

@@ -0,0 +1,130 @@
+import path from 'node:path';
+import { Octokit } from '@octokit/rest';
+import ky from 'ky';
+import PQueue from 'p-queue';
+import YAML from 'yaml';
+import argvUtils from '../utils/argv.js';
+import appConfig from '../utils/config.js';
+import githubUtils from '../utils/github.js';
+import logger from '../utils/logger.js';
+import mathUtils from '../utils/math.js';
+import stringUtils from '../utils/string.js';
+
+interface MirrorFileEntry {
+  orig: string;
+  mirror: string;
+  origStatus: boolean;
+}
+
+interface AssetToMirror {
+  url: string;
+  name: string | null;
+}
+
+let githubAuthCfg: any = null;
+let octoClient: Octokit | null = null;
+const networkQueue = new PQueue({ concurrency: appConfig.threadCount.network });
+
+const formatBytes = (size: number) =>
+  mathUtils.formatFileSize(size, {
+    decimals: 2,
+    decimalPadding: true,
+    unitVisible: true,
+    useBinaryUnit: true,
+    useBitUnit: false,
+    unit: null,
+  });
+
+async function checkMirrorFileDbStatus() {
+  logger.info('Checking mirrored files availability ...');
+  const outputDir = argvUtils.getArgv()['outputDir'];
+  const dbPath = path.join(outputDir, 'mirror_file_list.json');
+  if (!(await Bun.file(dbPath).exists())) return;
+
+  const db = (await Bun.file(dbPath).json()) as MirrorFileEntry[];
+
+  for (const entry of db) {
+    networkQueue.add(async () => {
+      try {
+        await ky.head(entry.orig, {
+          headers: { 'User-Agent': appConfig.network.userAgent.minimum },
+          timeout: appConfig.network.timeout,
+          retry: { limit: appConfig.network.retryCount },
+        });
+        entry.origStatus = true;
+      } catch {
+        if (entry.origStatus) logger.trace(`Orig inaccessible: ${entry.orig.split('/').pop()}`);
+        entry.origStatus = false;
+      }
+    });
+  }
+  await networkQueue.onIdle();
+  await Bun.write(dbPath, JSON.stringify(db, null, 2));
+}
+
+async function processMirrorQueue() {
+  const outputDir = argvUtils.getArgv()['outputDir'];
+  const dbPath = path.join(outputDir, 'mirror_file_list.json');
+  const pendingPath = path.join(outputDir, 'mirror_file_list_pending.json');
+
+  if (!(await Bun.file(pendingPath).exists())) {
+    logger.info('No pending assets to mirror');
+    return;
+  }
+
+  const db: MirrorFileEntry[] = (await Bun.file(dbPath).exists()) ? await Bun.file(dbPath).json() : [];
+  const pending: AssetToMirror[] = await Bun.file(pendingPath).json();
+
+  if (pending.length === 0) {
+    logger.info('Pending list is empty');
+    return;
+  }
+
+  logger.info(`Processing ${pending.length} pending assets ...`);
+
+  for (const { url, name } of pending) {
+    const origUrl = stringUtils.removeQueryStr(url);
+    if (!db.find((e) => e.orig.includes(origUrl))) {
+      await githubUtils.uploadAsset(octoClient, githubAuthCfg, url, name);
+      if (githubAuthCfg) {
+        db.push({
+          orig: origUrl,
+          mirror: `https://github.com/${githubAuthCfg.github.relArchive.owner}/${githubAuthCfg.github.relArchive.repo}/releases/download/${githubAuthCfg.github.relArchive.tag}/${name ?? new URL(url).pathname.split('/').pop() ?? ''}`,
+          origStatus: true,
+        });
+        await Bun.write(dbPath, JSON.stringify(db, null, 2));
+      }
+    }
+  }
+
+  // Clear pending list
+  await Bun.write(pendingPath, JSON.stringify([], null, 2));
+  logger.info('Mirroring process completed and pending list cleared');
+}
+
+async function mainCmdHandler() {
+  const authPath = 'config/config_auth.yaml';
+  if (await Bun.file(authPath).exists()) {
+    githubAuthCfg = YAML.parse(await Bun.file(authPath).text());
+    logger.info('Logging in to GitHub');
+    octoClient = new Octokit({ auth: githubAuthCfg.github.relArchive.token });
+  } else {
+    logger.error('GitHub authentication config not found');
+    return;
+  }
+
+  if (await githubUtils.checkIsActionRunning(githubAuthCfg)) {
+    logger.error('Duplicate execution detected (GitHub Action is already running)');
+    return;
+  }
+
+  await checkMirrorFileDbStatus();
+  await processMirrorQueue();
+
+  const relInfo = await githubUtils.getReleaseInfo(octoClient, githubAuthCfg);
+  if (relInfo) {
+    logger.info(`GitHub Releases total size: ${formatBytes(mathUtils.arrayTotal(relInfo.assets.map((a) => a.size)))}`);
+  }
+}
+
+export default mainCmdHandler;