Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@
"mysql2": "^3.11.0",
"openpgp": "^6.3.1",
"otplib": "^12.0.1",
"parse5": "^7.3.0",
"qrcode": "^1.5.4",
"rehype-katex": "7.0.1",
"rehype-raw": "^7.0.0",
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

71 changes: 58 additions & 13 deletions scripts/fetch-frontend.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
* 1. FRONTEND_DIST 环境变量:已构建好的 dist 目录路径(最快,CI 缓存场景)
* 2. FRONTEND_REPO 环境变量:本地官方前端仓库路径(自动 install + build)
* 3. 同级目录 ../OpenList-Frontend(monorepo 布局,自动探测,自动 install + build)
* 4. 默认:下载 npm 上【已发布】的 dist(版本取 registry 的 latest,
* 可用 FRONTEND_VERSION 固定)
* 5. FRONTEND_BUILD_FROM_SOURCE=1:从 Git 克隆前端 main 分支并现构建
* 4. 默认:下载 npm 上【已发布】且兼容 Worker 初始化协议的 dist(版本取
* registry 的 latest,可用 FRONTEND_VERSION 固定)
* 5. 发布版尚未包含 Worker 初始化协议,或 FRONTEND_BUILD_FROM_SOURCE=1:
* 从 Git 克隆前端 main 分支并现构建
*
* 为什么默认取「已发布 dist」而不是「克隆 main 现构建」:
* 前端产物是内容哈希文件名(/assets/index-XXXX.js),而 CDN(jsdelivr /
Expand All @@ -22,6 +23,10 @@
* 可用(等价 Go Release 版行为)。同时 stampFrontendVersion 会把该版本号写进
* index.html,使 ASSET_URLS 的 $version 正好解析到这份 dist 对应的版本。
*
* 已发布包可能落后于 Worker 后端。只有同时包含 /public/init_status 与 /@init
* 的产物才可使用;否则全新部署会只请求会被 503 拦截的 /public/settings,既不
* 显示初始化向导也无法创建管理员。遇到这种版本会自动改为构建前端 main。
*
* 用法:
* FRONTEND_DIST=/path/to/dist node scripts/fetch-frontend.mjs
* FRONTEND_REPO=../OpenList-Frontend node scripts/fetch-frontend.mjs
Expand Down Expand Up @@ -95,6 +100,32 @@ function requireDist(src) {
}
}

/**
* 已构建前端是否包含 Worker 首次初始化协议。
*
* Vite 会保留路由和 API 路径字符串,因此无需执行或反编译 bundle。两项必须
* 同时存在:init_status 负责识别空数据库,/@init 才能渲染创建管理员的向导。
*/
function supportsWorkerSetup(src) {
let hasStatus = false
let hasRoute = false
const pending = [src]
while (pending.length > 0 && (!hasStatus || !hasRoute)) {
const current = pending.pop()
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
const full = path.join(current, entry.name)
if (entry.isDirectory()) {
pending.push(full)
} else if (entry.isFile() && entry.name.endsWith(".js")) {
const code = fs.readFileSync(full, "utf-8")
hasStatus ||= code.includes("/public/init_status")
hasRoute ||= code.includes("/@init")
}
}
}
return hasStatus && hasRoute
}

function replaceDist(src) {
console.log(` Copying frontend dist: ${src} -> ${DEST}`)
fs.rmSync(DEST, { recursive: true, force: true })
Expand All @@ -113,7 +144,10 @@ function replaceDist(src) {
function stampFrontendVersion(src) {
try {
const pkg = JSON.parse(
fs.readFileSync(path.join(path.resolve(src, ".."), "package.json"), "utf-8"),
fs.readFileSync(
path.join(path.resolve(src, ".."), "package.json"),
"utf-8",
),
)
// 只信任官方前端包的版本号:FRONTEND_DIST 可能指向任意目录,
// 误读(例如 worker 自身 package.json 的 4.2.3)会戳出错误的 CDN 版本。
Expand Down Expand Up @@ -145,7 +179,9 @@ function stampFrontendVersion(src) {
function fetchI18n(repo) {
const langDir = path.join(repo, "src", "lang")
if (!fs.existsSync(langDir)) {
console.warn(" [fetch-frontend] repo missing src/lang, skipping i18n fetch")
console.warn(
" [fetch-frontend] repo missing src/lang, skipping i18n fetch",
)
return
}
const tmpTar = path.join(os.tmpdir(), `openlist-i18n-${process.pid}.tar.gz`)
Expand All @@ -172,16 +208,17 @@ function buildLocalRepo(repo) {
}
const pm = detectPackageManager(abs)
const cmd = resolvePmCommand(abs, pm)
const install = (extra = "") =>
run(`${cmd} install${extra}`, { cwd: abs })
const install = (extra = "") => run(`${cmd} install${extra}`, { cwd: abs })
try {
install()
} catch {
// 重试一次并加 --trust-lockfile:pnpm 11 默认对 lockfile 逐项重跑
// minimumReleaseAge / trustPolicy 供应链复核,registry manifest 缺少
// 平台子包时会误报(如 @crowdin/cli-*-arm64)。lockfile 来自刚克隆的
// 官方前端仓库(HTTPS + 官方分支),属于可信来源,跳过复核安全。
console.warn(" [fetch-frontend] pnpm install failed (lockfile supply-chain recheck or network issue), retrying once with --trust-lockfile...")
console.warn(
" [fetch-frontend] pnpm install failed (lockfile supply-chain recheck or network issue), retrying once with --trust-lockfile...",
)
install(" --trust-lockfile")
}
fetchI18n(abs)
Expand Down Expand Up @@ -239,7 +276,14 @@ async function fetchPublishedDist() {
run(`tar -xzf pkg.tgz package/dist package/package.json`, { cwd: tmp })
const src = path.join(tmp, "package", "dist")
requireDist(src)
if (!supportsWorkerSetup(src)) {
console.warn(
` Published frontend ${version} lacks the Worker setup protocol; building ${OFFICIAL_REPO_REF} instead`,
)
return false
}
replaceDist(src)
return true
} finally {
fs.rmSync(tmp, { recursive: true, force: true })
}
Expand Down Expand Up @@ -272,17 +316,18 @@ async function main() {
return
}

// 4. 下载 npm 上已发布的 dist(默认)
// 4. 下载 npm 上已发布且兼容 Worker 初始化协议的 dist(默认)
// 从 main 现构建的产物哈希与 CDN 不一致,会让路径 A 失效、npmmirror 之类的
// 镜像完全不可用(详见文件头),故默认改为取已发布产物。
if (process.env.FRONTEND_BUILD_FROM_SOURCE !== "1") {
await fetchPublishedDist()
return
if (await fetchPublishedDist()) return
}

// 5. 从 Git 克隆 main 并构建(FRONTEND_BUILD_FROM_SOURCE=1 时使用)
// 5. 从 Git 克隆 main 并构建(显式要求源码构建,或发布版尚未兼容 Worker)
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "openlist-frontend-"))
console.log(` Cloning official frontend: ${OFFICIAL_REPO_URL}#${OFFICIAL_REPO_REF}`)
console.log(
` Cloning official frontend: ${OFFICIAL_REPO_URL}#${OFFICIAL_REPO_REF}`,
)
try {
run(
// -c core.autocrlf=false:禁用克隆端的换行符转换。Windows 上 autocrlf
Expand Down
25 changes: 25 additions & 0 deletions src/backend/drivers/autoindex/driver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,31 @@ test("parseAutoIndexHTML extracts files, dirs, size and modified", () => {
assert.equal(nodes[1].isDir, true)
})

test("parseAutoIndexHTML tolerates malformed Apache directory HTML", () => {
// archive.apache.org/dist/tomcat/ 实际返回过这种重复 html/body/pre、只关闭一层
// pre 的页面。浏览器与 Go htmlquery 都能解析,AutoIndex 也不能按严格 XML 拒绝。
const html = `<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html><head><title>Index</title></head><body>
<html><head><title>Index</title></head><body><pre>
<pre><a href="/dist/">Parent Directory</a>
<a href="tomcat-10/">tomcat-10/</a> 2026-09-15 10:54 -
</pre></body></html>`

const nodes = parseAutoIndexHTML(
html,
"//pre/a",
"@href",
"string(following-sibling::text()[1])",
"string(following-sibling::text()[2])",
["Parent Directory"],
)

const tomcat = nodes.find((node) => node.name === "tomcat-10")
assert.ok(tomcat)
assert.equal(tomcat.url, "tomcat-10/")
assert.equal(tomcat.isDir, true)
})

test("normalizeAutoIndexAddition fills defaults", () => {
const a = normalizeAutoIndexAddition({ url: "example.com/files" })
assert.equal(a.url, "https://example.com/files/")
Expand Down
8 changes: 7 additions & 1 deletion src/backend/drivers/autoindex/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const DefaultItemXPath = "//pre/a"
const DefaultNameXPath = "@href"
const DefaultSizeXPath = "string(following-sibling::text()[1])"
const DefaultModifiedXPath = "string(following-sibling::text()[2])"
const FETCH_TIMEOUT_MS = 30_000

export function normalizeAutoIndexAddition(a: any): AutoIndexAddition {
const norm = { ...(a || {}) } as any
Expand Down Expand Up @@ -78,7 +79,12 @@ export class AutoIndexDriver implements StorageDriver {

async list(virtualPath: string, physicalPath: string): Promise<FileItem[]> {
const baseURL = this.buildDirURL(physicalPath)
const res = await fetch(baseURL)
// 边缘运行时通常会等到平台级超时(EdgeOne 为 120 秒)才中止不可达的
// 上游请求。显式限制单次目录读取,避免一个失联的 AutoIndex 挂载长期占用
// 实例并拖累同一服务的其他请求。
const res = await fetch(baseURL, {
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
})
if (!res.ok) {
throw new Error(`Failed to fetch ${baseURL}: HTTP ${res.status}`)
}
Expand Down
35 changes: 23 additions & 12 deletions src/backend/drivers/autoindex/util.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// AutoIndex utility functions
import * as xpath from "xpath"
import { DOMParser } from "@xmldom/xmldom"
import { parse, serialize } from "parse5"
import { AutoIndexNode } from "./types"

/**
Expand All @@ -20,19 +21,30 @@ function extractXPathValue(raw: unknown): string | undefined {
}

// 将宽松 HTML 清洗为可被 XML 解析器(xmldom)解析的良构 XML:
// 1) 移除 DOCTYPE 与注释;2) 把 void 标签(<hr>/<meta>/<br> 等)转为自闭合。
// 这样用 text/xml 解析后节点无命名空间,用户配置的无前缀 XPath(如 //pre/a)
// 才能正常匹配——与 Go 侧 htmlquery(golang.org/x/net/html)的行为保持一致。
// 注意:若直接用 text/html 解析,xmldom 会注入 XHTML 命名空间,导致 //pre/a 匹配不到。
// 1) 先用 parse5 按 HTML5 容错规则修复未闭合/错位/重复标签;
// 2) 移除 XML 不需要的 DOCTYPE、注释和 script/style;
// 3) 把 void 标签(<hr>/<meta>/<br> 等)转为自闭合。
//
// 不能直接把远端 HTML 交给 xmldom 的 XML 模式:真实目录页常包含浏览器可以
// 正常处理的非严格 HTML。例如 archive.apache.org/dist/tomcat/ 同时有重复的
// html/body/pre 标签,旧实现会抛 "Opening and ending tag mismatch"。parse5 与
// Go 版 htmlquery 底层的 HTML parser 一样会容错,再转 XML 后仍可继续使用用户
// 配置的无前缀 XPath(如 //pre/a)。
//
// 注意:若直接用 xmldom 的 text/html 模式,它会注入 XHTML 命名空间,导致
// //pre/a 匹配不到。
const HTML_VOID_TAGS =
"area|base|br|col|embed|hr|img|input|link|meta|param|source|track|wbr"

function htmlToXml(html: string): string {
let s = html
let s = serialize(parse(html))
s = s.replace(/<!DOCTYPE[^>]*>/gi, "")
s = s.replace(/<!--[\s\S]*?-->/g, "")
// script/style 的 raw text 可以包含 XML 非法的裸 <、&;AutoIndex XPath 不会
// 依赖这些内容,移除可避免第二阶段 XML 解析被无关脚本或样式破坏。
s = s.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, "")
s = s.replace(
new RegExp(`<(${HTML_VOID_TAGS})([^>]*?)/?>`, "gi"),
new RegExp(`<(${HTML_VOID_TAGS})([^>]*?)/?\\s*>`, "gi"),
"<$1$2/>",
)
return s
Expand All @@ -44,7 +56,7 @@ export function parseAutoIndexHTML(
nameXPath: string,
sizeXPath: string,
modifiedXPath: string,
ignoreNames: string[]
ignoreNames: string[],
): AutoIndexNode[] {
// @xmldom/xmldom >= 0.9 已废弃 errorHandler,改用 onError 回调
const doc = new DOMParser({
Expand Down Expand Up @@ -146,10 +158,7 @@ export function parseSize(sizeStr: string): number {
return Math.round(num * mul)
}

export function parseTime(
timeStr: string,
format: string
): string {
export function parseTime(timeStr: string, format: string): string {
if (!timeStr) return new Date().toISOString()

try {
Expand All @@ -161,7 +170,9 @@ export function parseTime(
const isoMatch = timeStr.match(/(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2})/)
if (isoMatch) {
const [, year, month, day, hour, minute] = isoMatch
return new Date(`${year}-${month}-${day}T${hour}:${minute}:00`).toISOString()
return new Date(
`${year}-${month}-${day}T${hour}:${minute}:00`,
).toISOString()
}

// Try common formats
Expand Down
Loading
Loading