diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eedba56..31bd337 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ permissions: checks: write contents: write on: - # Scheduled trigger is disabled by default + # Scheduled trigger is disabled by default (use cron-job) # Uncommenting the following two lines will enable it # schedule: # - cron: '0 16 * * *' @@ -32,20 +32,27 @@ jobs: max_attempts: 5 timeout_minutes: 10 command: bun run src/main.ts archive - - name: Format output data + - name: Format output folder run: bun x oxfmt output - # - name: Git staging, commit, push - # continue-on-error: true - # run: | - # git config user.name "github-actions[bot]" - # git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - # git add output - # git commit -m '[Auto] API update' - # git push - name: Git commit and push uses: iarekylew00t/verified-bot-commit@v2 with: message: "[Auto] API update" files: | - output + output/akEndfield + output/raw + output/mirror_file_list.json + if-no-commit: info + - name: Run GitHub mirror upload + run: bun run src/main.ts ghMirrorUpload + continue-on-error: true + - name: Format output folder + run: bun x oxfmt output + - name: Git commit and push + uses: iarekylew00t/verified-bot-commit@v2 + with: + message: "[Auto] API update" + files: | + output/mirror_file_list.json + output/mirror_file_list_pending.json if-no-commit: info diff --git a/output/mirror_file_list.json b/output/mirror_file_list.json index dedfde3..ca84fb1 100644 --- a/output/mirror_file_list.json +++ b/output/mirror_file_list.json @@ -974,4 +974,4 @@ "mirror": "https://github.com/AetherArchive/beyond-hg-archive/releases/download/tag/1.0.14_bJBg3b40frDq9bOB_patches_Beyond_Release_v1d0-Rel-cn-5157154-10_prod_obt_bilibili_1_0_13.zip", "origStatus": false } -] +] \ No newline at end of file diff --git a/output/mirror_file_list_pending.json b/output/mirror_file_list_pending.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/output/mirror_file_list_pending.json @@ -0,0 +1 @@ +[] diff --git a/src/cmd.ts b/src/cmd.ts index 023b2d3..13b497c 100644 --- a/src/cmd.ts +++ b/src/cmd.ts @@ -42,6 +42,22 @@ async function parseCommand() { }, wrapHandler(cmds.archive), ) + .command( + ['ghMirrorUpload'], + 'Upload pending large binary file to GitHub mirror', + (yargs) => { + yargs.options({ + 'output-dir': { + alias: ['o'], + desc: 'Output root directory', + default: path.resolve('output'), + normalize: true, + type: 'string', + }, + }); + }, + wrapHandler(cmds.ghMirrorUpload), + ) .command( ['authTest [token] [email] [password]'], 'Auth and gacha fetch test command', diff --git a/src/cmds.ts b/src/cmds.ts index 31a01d4..aa2b2bd 100644 --- a/src/cmds.ts +++ b/src/cmds.ts @@ -1,7 +1,9 @@ import archive from './cmds/archive.js'; import authTest from './cmds/authTest.js'; +import ghMirrorUpload from './cmds/ghMirrorUpload.js'; export default { authTest, archive, + ghMirrorUpload, }; diff --git a/src/cmds/archive.ts b/src/cmds/archive.ts index 5af9e91..a395328 100644 --- a/src/cmds/archive.ts +++ b/src/cmds/archive.ts @@ -1,14 +1,11 @@ import path from 'node:path'; -import { Octokit } from '@octokit/rest'; import ky, { HTTPError } from 'ky'; import { DateTime } from 'luxon'; import PQueue from 'p-queue'; import semver from 'semver'; -import YAML from 'yaml'; import apiUtils from '../utils/api/index.js'; import argvUtils from '../utils/argv.js'; import appConfig from '../utils/config.js'; -import githubUtils from '../utils/github.js'; import logger from '../utils/logger.js'; import mathUtils from '../utils/math.js'; import stringUtils from '../utils/string.js'; @@ -23,6 +20,12 @@ interface StoredData { updatedAt: string; } +interface MirrorFileEntry { + orig: string; + mirror: string; + origStatus: boolean; +} + interface GameTarget { name: string; region: 'os' | 'cn'; @@ -41,20 +44,12 @@ interface LauncherTarget { channel: number; } -interface MirrorFileEntry { - orig: string; - mirror: string; - origStatus: boolean; -} - interface AssetToMirror { url: string; name: string | null; } // Global/Shared State -let githubAuthCfg: any = null; -let octoClient: Octokit | null = null; const assetsToMirror: AssetToMirror[] = []; const networkQueue = new PQueue({ concurrency: appConfig.threadCount.network }); @@ -572,64 +567,7 @@ async function fetchAndSaveLatestWebApis(gameTargets: GameTarget[]) { await networkQueue.onIdle(); } -// Mirroring and Cleanup -async function checkMirrorFileDbStatus() { - logger.info('Checking mirrored files availability ...'); - const dbPath = path.join(argvUtils.getArgv()['outputDir'], 'mirror_file_list.json'); - const db = (await Bun.file(dbPath).json()) as MirrorFileEntry[]; - - for (const entry of db) { - networkQueue.add(async () => { - try { - await ky.head(entry.orig, { - headers: { 'User-Agent': appConfig.network.userAgent.minimum }, - timeout: appConfig.network.timeout, - retry: { limit: appConfig.network.retryCount }, - }); - entry.origStatus = true; - } catch { - if (entry.origStatus) logger.trace(`Orig inaccessible: ${entry.orig.split('/').pop()}`); - entry.origStatus = false; - } - }); - } - await networkQueue.onIdle(); - await Bun.write(dbPath, JSON.stringify(db)); -} - -async function processMirrorQueue() { - const dbPath = path.join(argvUtils.getArgv()['outputDir'], 'mirror_file_list.json'); - const db = (await Bun.file(dbPath).json()) as MirrorFileEntry[]; - - for (const { url, name } of assetsToMirror) { - const origUrl = stringUtils.removeQueryStr(url); - if (!db.find((e) => e.orig.includes(origUrl))) { - await githubUtils.uploadAsset(octoClient, githubAuthCfg, url, name); - if (githubAuthCfg) { - db.push({ - orig: origUrl, - mirror: `https://github.com/${githubAuthCfg.github.relArchive.owner}/${githubAuthCfg.github.relArchive.repo}/releases/download/${githubAuthCfg.github.relArchive.tag}/${name ?? new URL(url).pathname.split('/').pop() ?? ''}`, - origStatus: true, - }); - await Bun.write(dbPath, JSON.stringify(db)); - } - } - } -} - async function mainCmdHandler() { - const authPath = 'config/config_auth.yaml'; - if (await Bun.file(authPath).exists()) { - githubAuthCfg = YAML.parse(await Bun.file(authPath).text()); - logger.info('Logging in to GitHub'); - octoClient = new Octokit({ auth: githubAuthCfg.github.relArchive.token }); - } - - if (await githubUtils.checkIsActionRunning(githubAuthCfg)) { - logger.error('Duplicate execution detected'); - return; - } - const cfg = appConfig.network.api.akEndfield; const gameTargets: GameTarget[] = [ { @@ -701,12 +639,35 @@ async function mainCmdHandler() { await fetchAndSaveLatestLauncher(launcherTargets); await fetchAndSaveAllGameResRawData(gameTargets); - await checkMirrorFileDbStatus(); - await processMirrorQueue(); + // Save pending assets to mirror + const outputDir = argvUtils.getArgv()['outputDir']; + const pendingPath = path.join(outputDir, 'mirror_file_list_pending.json'); + const dbPath = path.join(outputDir, 'mirror_file_list.json'); - const relInfo = await githubUtils.getReleaseInfo(octoClient, githubAuthCfg); - if (relInfo) { - logger.info(`GitHub Releases total size: ${formatBytes(mathUtils.arrayTotal(relInfo.assets.map((a) => a.size)))}`); + let pendingData: AssetToMirror[] = []; + if (await Bun.file(pendingPath).exists()) { + pendingData = await Bun.file(pendingPath).json(); + } + const db: MirrorFileEntry[] = (await Bun.file(dbPath).exists()) ? await Bun.file(dbPath).json() : []; + let addedCount = 0; + + const uniqueAssetsToMirror = assetsToMirror.filter( + (asset, index, self) => index === self.findIndex((t) => t.url === asset.url), + ); + + for (const asset of uniqueAssetsToMirror) { + const origUrl = stringUtils.removeQueryStr(asset.url); + const dbExists = db.some((e) => e.orig.includes(origUrl)); + const pendingExists = pendingData.some((e) => stringUtils.removeQueryStr(e.url) === origUrl); + if (!dbExists && !pendingExists) { + pendingData.push(asset); + addedCount++; + } + } + + if (addedCount > 0) { + logger.info(`Saved ${addedCount} new assets to mirror pending list`); + await Bun.write(pendingPath, JSON.stringify(pendingData, null, 2)); } } diff --git a/src/cmds/ghMirrorUpload.ts b/src/cmds/ghMirrorUpload.ts new file mode 100644 index 0000000..3ccb3b1 --- /dev/null +++ b/src/cmds/ghMirrorUpload.ts @@ -0,0 +1,130 @@ +import path from 'node:path'; +import { Octokit } from '@octokit/rest'; +import ky from 'ky'; +import PQueue from 'p-queue'; +import YAML from 'yaml'; +import argvUtils from '../utils/argv.js'; +import appConfig from '../utils/config.js'; +import githubUtils from '../utils/github.js'; +import logger from '../utils/logger.js'; +import mathUtils from '../utils/math.js'; +import stringUtils from '../utils/string.js'; + +interface MirrorFileEntry { + orig: string; + mirror: string; + origStatus: boolean; +} + +interface AssetToMirror { + url: string; + name: string | null; +} + +let githubAuthCfg: any = null; +let octoClient: Octokit | null = null; +const networkQueue = new PQueue({ concurrency: appConfig.threadCount.network }); + +const formatBytes = (size: number) => + mathUtils.formatFileSize(size, { + decimals: 2, + decimalPadding: true, + unitVisible: true, + useBinaryUnit: true, + useBitUnit: false, + unit: null, + }); + +async function checkMirrorFileDbStatus() { + logger.info('Checking mirrored files availability ...'); + const outputDir = argvUtils.getArgv()['outputDir']; + const dbPath = path.join(outputDir, 'mirror_file_list.json'); + if (!(await Bun.file(dbPath).exists())) return; + + const db = (await Bun.file(dbPath).json()) as MirrorFileEntry[]; + + for (const entry of db) { + networkQueue.add(async () => { + try { + await ky.head(entry.orig, { + headers: { 'User-Agent': appConfig.network.userAgent.minimum }, + timeout: appConfig.network.timeout, + retry: { limit: appConfig.network.retryCount }, + }); + entry.origStatus = true; + } catch { + if (entry.origStatus) logger.trace(`Orig inaccessible: ${entry.orig.split('/').pop()}`); + entry.origStatus = false; + } + }); + } + await networkQueue.onIdle(); + await Bun.write(dbPath, JSON.stringify(db, null, 2)); +} + +async function processMirrorQueue() { + const outputDir = argvUtils.getArgv()['outputDir']; + const dbPath = path.join(outputDir, 'mirror_file_list.json'); + const pendingPath = path.join(outputDir, 'mirror_file_list_pending.json'); + + if (!(await Bun.file(pendingPath).exists())) { + logger.info('No pending assets to mirror'); + return; + } + + const db: MirrorFileEntry[] = (await Bun.file(dbPath).exists()) ? await Bun.file(dbPath).json() : []; + const pending: AssetToMirror[] = await Bun.file(pendingPath).json(); + + if (pending.length === 0) { + logger.info('Pending list is empty'); + return; + } + + logger.info(`Processing ${pending.length} pending assets ...`); + + for (const { url, name } of pending) { + const origUrl = stringUtils.removeQueryStr(url); + if (!db.find((e) => e.orig.includes(origUrl))) { + await githubUtils.uploadAsset(octoClient, githubAuthCfg, url, name); + if (githubAuthCfg) { + db.push({ + orig: origUrl, + mirror: `https://github.com/${githubAuthCfg.github.relArchive.owner}/${githubAuthCfg.github.relArchive.repo}/releases/download/${githubAuthCfg.github.relArchive.tag}/${name ?? new URL(url).pathname.split('/').pop() ?? ''}`, + origStatus: true, + }); + await Bun.write(dbPath, JSON.stringify(db, null, 2)); + } + } + } + + // Clear pending list + await Bun.write(pendingPath, JSON.stringify([], null, 2)); + logger.info('Mirroring process completed and pending list cleared'); +} + +async function mainCmdHandler() { + const authPath = 'config/config_auth.yaml'; + if (await Bun.file(authPath).exists()) { + githubAuthCfg = YAML.parse(await Bun.file(authPath).text()); + logger.info('Logging in to GitHub'); + octoClient = new Octokit({ auth: githubAuthCfg.github.relArchive.token }); + } else { + logger.error('GitHub authentication config not found'); + return; + } + + if (await githubUtils.checkIsActionRunning(githubAuthCfg)) { + logger.error('Duplicate execution detected (GitHub Action is already running)'); + return; + } + + await checkMirrorFileDbStatus(); + await processMirrorQueue(); + + const relInfo = await githubUtils.getReleaseInfo(octoClient, githubAuthCfg); + if (relInfo) { + logger.info(`GitHub Releases total size: ${formatBytes(mathUtils.arrayTotal(relInfo.assets.map((a) => a.size)))}`); + } +} + +export default mainCmdHandler;