From 40bb7ea09518c30f9aa11480ee682a7d762b3d04 Mon Sep 17 00:00:00 2001 From: ermaozi Date: Tue, 8 Sep 2026 08:03:12 +0800 Subject: [PATCH] feat: deploy GitHub subscription sync to Cloudflare with ermao branding --- .github/workflows/main.yml | 2 + .gitignore | 2 + README.md | 8 +- main.py | 20 +++++ tests/test_subscriptions.py | 8 +- worker/sub.mjs | 163 ++++++++++++++++++++++++++++++++++++ worker/sub.test.mjs | 95 +++++++++++++++++++++ worker/wrangler.jsonc | 29 +++++++ 8 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 worker/sub.mjs create mode 100644 worker/sub.test.mjs create mode 100644 worker/wrangler.jsonc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 77761dd..949572a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,6 +33,8 @@ jobs: python -m pip install -r requirements.txt - name: 回归检查 run: python -m unittest discover -s tests + - name: Worker 回归检查 + run: node --test worker/sub.test.mjs - name: 执行任务 run: | python main.py diff --git a/.gitignore b/.gitignore index 1e248ad..8ce73ad 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ venv/ .venv/ __pycache__/ *.pyc + +.wrangler/ diff --git a/README.md b/README.md index e069d9b..bcd2161 100644 --- a/README.md +++ b/README.md @@ -68,5 +68,11 @@ https://www.ermao.net/sub/v2ray/ermao.net - `subscribe/unreachable/`：TCP 连接或 DNS 解析失败的节点。 - `subscribe/untested/`：UDP/QUIC 协议、无法解析地址或非公网地址，不判定为不可用。 -`subscribe/health.json` 保存检测时间和分类数量。 +在线分类订阅（将链接中的 `clash` 换成 `v2ray` 可获取对应格式）： + +- [TCP 可连接](https://www.ermao.net/sub/reachable/clash/ermao.net) +- [连接失败](https://www.ermao.net/sub/unreachable/clash/ermao.net) +- [未检测](https://www.ermao.net/sub/untested/clash/ermao.net) + +[检测报告](https://www.ermao.net/sub/health.json)保存检测时间和分类数量。 这不是代理认证、出口访问或速度测试；可连接不等于代理可用，失败也可能是运行机器的网络限制。 diff --git a/main.py b/main.py index b6f8d66..69a68b3 100644 --- a/main.py +++ b/main.py @@ -170,6 +170,26 @@ def _merge_clash(sources): [renamed.get(member, member) for member in members] + [p["name"] for p in proxies] )) config["proxies"] = proxies + # 仅给前两个选择组加入站点标识，同时保持规则和组间引用一致。 + groups = config.get("proxy-groups", []) + branded = {} + used_names = {p["name"] for p in proxies} | {g["name"] for g in groups} + for group in [g for g in groups if g.get("type") in {"select", "url-test"}][:2]: + old = group["name"] + if "ermao.net" in old: + continue + name = f"ermao.net | {old}" + while name in used_names: + name += " ·" + branded[old] = name + used_names.add(name) + group["name"] = name + for group in groups: + if "proxies" in group: + group["proxies"] = [branded.get(name, name) for name in group["proxies"]] + if "rules" in config: + config["rules"] = [",".join(branded.get(part, part) for part in rule.split(",")) + for rule in config["rules"]] return yaml.safe_dump(config, allow_unicode=True, sort_keys=False) diff --git a/tests/test_subscriptions.py b/tests/test_subscriptions.py index 263d9a1..65f405c 100644 --- a/tests/test_subscriptions.py +++ b/tests/test_subscriptions.py @@ -55,14 +55,16 @@ class SubscriptionTests(unittest.TestCase): os.chdir(previous) def test_merge_deduplicates_and_keeps_groups_resolvable(self): - first = CLASH + 'proxy-groups:\n - {name: select, type: select, proxies: [test, DIRECT]}\nrules: ["MATCH,select"]\n' + first = CLASH + 'proxy-groups:\n - {name: select, type: select, proxies: [test, DIRECT, auto]}\n - {name: auto, type: url-test, proxies: [test]}\nrules: ["MATCH,select"]\n' duplicate = CLASH.replace('name: test', 'name: another') collision = CLASH.replace('example.com', 'other.example') data = yaml.safe_load(main._merge_clash([('original', first), ('NoMoreWalls', duplicate), ('ProxyPool', collision)])) names = [p['name'] for p in data['proxies']] self.assertEqual(names, ['test', 'ProxyPool | test']) - self.assertEqual(data['proxy-groups'][0]['proxies'], ['test', 'DIRECT', 'ProxyPool | test']) - self.assertEqual(data['rules'], ['MATCH,select']) + self.assertEqual(data['proxy-groups'][0]['proxies'], ['test', 'DIRECT', 'ermao.net | auto', 'ProxyPool | test']) + self.assertEqual(data['proxy-groups'][0]['name'], 'ermao.net | select') + self.assertEqual(data['proxy-groups'][1]['name'], 'ermao.net | auto') + self.assertEqual(data['rules'], ['MATCH,ermao.net | select']) def test_direct_sources_work_when_rss_fails(self): session = Mock() diff --git a/worker/sub.mjs b/worker/sub.mjs new file mode 100644 index 0000000..0c32c6d --- /dev/null +++ b/worker/sub.mjs @@ -0,0 +1,163 @@ +const REPOSITORY = 'ermaozi/get_subscribe'; +const PREFIX = 'files/subscribe/'; +const CACHE_SECONDS = 300; +const MAX_FILE_BYTES = 4 * 1024 * 1024; +const FILES = ['clash.yml', 'v2ray.txt', ...['reachable', 'unreachable', 'untested'] + .flatMap(group => [`${group}/clash.yml`, `${group}/v2ray.txt`]), 'health.json']; + +function json(body, status = 200) { + return Response.json(body, { status, headers: { 'Cache-Control': 'no-store' } }); +} + +function fileForPath(path) { + if (path === '/sub/health.json') return 'health.json'; + const match = path.match(/^\/sub\/(?:(reachable|unreachable|untested)\/)?(clash|v2ray)\/ermao\.net$/); + return match ? `${match[1] ? match[1] + '/' : ''}${match[2] === 'clash' ? 'clash.yml' : 'v2ray.txt'}` : null; +} + +function contentType(file) { + if (file.endsWith('.json')) return 'application/json; charset=utf-8'; + return file.endsWith('.yml') ? 'text/yaml; charset=utf-8' : 'text/plain; charset=utf-8'; +} + +async function fetchText(url, limit = MAX_FILE_BYTES) { + const response = await fetch(url, { + headers: { 'User-Agent': 'ermao-subscription-sync' }, + redirect: 'manual', signal: AbortSignal.timeout(20000), + cf: { cacheTtl: 0, cacheEverything: false }, + }); + if (response.status !== 200) { + await response.body?.cancel(); + throw new Error(`Upstream HTTP ${response.status}`); + } + // 只读取有明确大小上限的文本，防止异常响应耗尽 Worker 内存。 + const reader = response.body?.getReader(); + if (!reader) return { text: '', bytes: 0 }; + const decoder = new TextDecoder('utf-8', { fatal: true }); + let text = '', bytes = 0; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + bytes += chunk.value.byteLength; + if (bytes > limit) throw new Error('Upstream file exceeds size limit'); + text += decoder.decode(chunk.value, { stream: true }); + } + text += decoder.decode(); + return { text, bytes }; + } finally { + await reader.cancel(); + } +} + +function validate(file, text) { + if (file === 'health.json') { + const report = JSON.parse(text); + if (!Number.isFinite(Date.parse(report.checked_at)) || report.method !== 'TCP connect') { + throw new Error('Invalid health report'); + } + for (const group of ['reachable', 'unreachable', 'untested']) { + for (const kind of ['clash', 'v2ray']) { + const count = report.counts?.[group]?.[kind]; + if (!Number.isInteger(count) || count < 0) throw new Error('Invalid health counts'); + } + } + return; + } + // YAML 结构已由主仓库回归测试和采集程序检查；此处防止错误页覆盖订阅。 + if (file.endsWith('.yml')) { + if (!/^proxies:\s/m.test(text)) throw new Error('Invalid Clash artifact'); + } else { + const nodes = text.trim() ? text.trim().split(/\r?\n/) : []; + if ((!file.includes('/') && !nodes.length) || nodes.some(node => + !/^(vmess|vless|trojan|ss|ssr|hysteria2?|hy2|tuic|anytls|mieru|https?|socks[45]?):\/\/\S+$/.test(node))) { + throw new Error('Invalid V2Ray artifact'); + } + } +} + +export async function refreshAll(env) { + if (!env.SUBSCRIBE_BUCKET) throw new Error('Missing subscription storage'); + const ref = await fetchText(`https://api.github.com/repos/${REPOSITORY}/git/ref/heads/main`, 16384); + const commit = JSON.parse(ref.text).object?.sha; + if (!/^[a-f0-9]{40}$/.test(commit)) throw new Error('Invalid upstream commit'); + const previous = await env.SUBSCRIBE_BUCKET.head(PREFIX + 'health.json'); + if (previous?.customMetadata?.commit === commit) { + return { ok: true, commit, files: FILES.length, unchanged: true, updatedAt: previous.customMetadata.updatedAt }; + } + const downloads = []; + let totalBytes = 0; + // 同一次同步固定到同一提交，所有文件验证完成后才写入 R2。 + for (const file of FILES) { + const url = `https://raw.githubusercontent.com/${REPOSITORY}/${commit}/subscribe/${file}`; + const body = await fetchText(url); + totalBytes += body.bytes; + if (totalBytes > 16 * 1024 * 1024) throw new Error('Subscription batch exceeds size limit'); + validate(file, body.text); + downloads.push({ file, url, text: body.text }); + } + const updatedAt = new Date().toISOString(); + for (const { file, url, text } of downloads) { + await env.SUBSCRIBE_BUCKET.put(PREFIX + file, text, { + httpMetadata: { contentType: contentType(file), cacheControl: `public, max-age=0, s-maxage=${CACHE_SECONDS}` }, + customMetadata: { commit, source: url, updatedAt }, + }); + } + console.log(JSON.stringify({ event: 'subscription_sync', commit, files: downloads.length, bytes: totalBytes })); + return { ok: true, commit, files: downloads.length, updatedAt }; +} + +async function serve(request, env, file) { + let object = await env.SUBSCRIBE_BUCKET.get(PREFIX + file); + if (!object) { + await refreshAll(env); + object = await env.SUBSCRIBE_BUCKET.get(PREFIX + file); + } + if (!object) throw new Error('Subscription not available'); + const headers = new Headers({ + 'Content-Type': contentType(file), 'Cache-Control': `public, max-age=0, s-maxage=${CACHE_SECONDS}`, + 'X-Source': 'r2', 'ETag': object.httpEtag, + }); + if (file !== 'health.json') { + const group = file.includes('/') ? file.split('/')[0] : 'all'; + const label = { all: '免费订阅', reachable: 'TCP 可连接', unreachable: '连接失败', untested: '未检测' }[group]; + const title = `ermao.net · ${label}`; + headers.set('Profile-Title', 'base64:' + btoa(String.fromCharCode(...new TextEncoder().encode(title)))); + headers.set('Profile-Web-Page-Url', 'https://www.ermao.net/'); + headers.set('Support-Url', 'https://www.ermao.net/'); + headers.set('Content-Disposition', `attachment; filename="ermao.net-${file.replaceAll('/', '-')}"`); + } + const metadata = object.customMetadata || {}; + if (/^[a-f0-9]{40}$/.test(metadata.commit)) headers.set('X-Git-Commit', metadata.commit); + if (metadata.updatedAt) headers.set('X-Updated-At', metadata.updatedAt.replace(/[\r\n]/g, '')); + if (request.headers.get('If-None-Match') === object.httpEtag) return new Response(null, { status: 304, headers }); + return new Response(request.method === 'HEAD' ? null : object.body, { headers }); +} + +export default { + async fetch(request, env) { + if (!['GET', 'HEAD'].includes(request.method)) return new Response('Method Not Allowed', { status: 405, headers: { Allow: 'GET, HEAD' } }); + const path = new URL(request.url).pathname.toLowerCase(); + if (['/', '/health', '/sub/health'].includes(path)) { + return json({ ok: true, service: 'subscribe-worker', version: 'github-sync-v1', endpoints: [ + '/sub/clash/ermao.net', '/sub/v2ray/ermao.net', '/sub/reachable/clash/ermao.net', '/sub/health.json', + ] }); + } + const refresh = path.match(/^\/sub\/refresh\/(all|clash|v2ray)$/); + const file = refresh && refresh[1] !== 'all' ? `${refresh[1] === 'clash' ? 'clash.yml' : 'v2ray.txt'}` : fileForPath(path); + if (!refresh && !file) return new Response('Not Found', { status: 404 }); + try { + if (refresh) { + const result = await refreshAll(env); + if (refresh[1] === 'all') return json(result); + } + return await serve(request, env, file); + } catch (error) { + console.error(JSON.stringify({ event: 'subscription_error', message: String(error.message || error) })); + return json({ ok: false, error: 'Subscription refresh or storage unavailable' }, 502); + } + }, + async scheduled(_controller, env, ctx) { + ctx.waitUntil(refreshAll(env)); + }, +}; diff --git a/worker/sub.test.mjs b/worker/sub.test.mjs new file mode 100644 index 0000000..579bb4c --- /dev/null +++ b/worker/sub.test.mjs @@ -0,0 +1,95 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import worker, { refreshAll } from './sub.mjs'; + +const commit = 'a'.repeat(40); +const clash = 'proxies:\n - {name: test, type: ss, server: example.com, port: 443}\n'; +const report = JSON.stringify({ checked_at: '2026-09-08T00:00:00Z', method: 'TCP connect', counts: { + reachable: { clash: 1, v2ray: 1 }, unreachable: { clash: 0, v2ray: 0 }, untested: { clash: 0, v2ray: 0 }, +} }); + +function storage() { + const objects = new Map(); + return { objects, SUBSCRIBE_BUCKET: { + async head(key) { return objects.get(key) || null; }, + async get(key) { + const item = objects.get(key); + return item ? { ...item, body: new Response(item.text).body, httpEtag: '"etag"' } : null; + }, + async put(key, text, options) { objects.set(key, { text, ...options }); }, + } }; +} + +function upstream(url) { + if (url.startsWith('https://api.github.com/')) return Response.json({ object: { sha: commit } }); + assert.ok(url.startsWith(`https://raw.githubusercontent.com/ermaozi/get_subscribe/${commit}/subscribe/`)); + if (url.endsWith('health.json')) return new Response(report); + const empty = /\/(unreachable|untested)\//.test(url); + return new Response(url.endsWith('.yml') ? (empty ? 'proxies: []\n' : clash) : (empty ? '' : 'vmess://test\n')); +} + +test('syncs one commit, all routes serve R2, unchanged revision does not download again', async t => { + const env = storage(); + const fetch = t.mock.method(globalThis, 'fetch', async url => upstream(url)); + const result = await refreshAll(env); + assert.equal(result.files, 9); + assert.equal(env.objects.size, 9); + assert.equal(fetch.mock.callCount(), 10); + for (const group of ['', 'reachable/', 'unreachable/', 'untested/']) { + for (const kind of ['clash', 'v2ray']) { + const response = await worker.fetch(new Request(`https://www.ermao.net/sub/${group}${kind}/ermao.net`), env); + assert.equal(response.status, 200); + assert.equal(response.headers.get('X-Git-Commit'), commit); + const title = Buffer.from(response.headers.get('Profile-Title').slice(7), 'base64').toString('utf-8'); + assert.match(title, /^ermao.net · /); + assert.equal(response.headers.get('Profile-Web-Page-Url'), 'https://www.ermao.net/'); + assert.match(response.headers.get('Content-Disposition'), /ermao\.net-/); + await response.text(); + } + } + const health = await worker.fetch(new Request('https://www.ermao.net/sub/health.json'), env); + assert.deepEqual(await health.json(), JSON.parse(report)); + assert.equal((await refreshAll(env)).unchanged, true); + assert.equal(fetch.mock.callCount(), 11); +}); + +test('invalid upstream file leaves existing objects unchanged; reads work during outage', async t => { + const env = storage(); + env.objects.set('files/subscribe/clash.yml', { text: clash, customMetadata: {} }); + t.mock.method(globalThis, 'fetch', async url => url.endsWith('health.json') ? new Response('error') : upstream(url)); + await assert.rejects(refreshAll(env)); + assert.equal(env.objects.size, 1); + assert.equal(env.objects.get('files/subscribe/clash.yml').text, clash); + const response = await worker.fetch(new Request('https://www.ermao.net/sub/clash/ermao.net'), env); + assert.equal(response.status, 200); + assert.equal(await response.text(), clash); +}); + +test('rejects invalid commits, excessive responses, unknown paths and unsupported methods', async t => { + const env = storage(); + const fetch = t.mock.method(globalThis, 'fetch', async () => Response.json({ object: { sha: '../other' } })); + await assert.rejects(refreshAll(env), /Invalid upstream commit/); + fetch.mock.mockImplementation(async url => url.startsWith('https://api.github.com/') + ? upstream(url) : new Response('x'.repeat(4 * 1024 * 1024 + 1))); + await assert.rejects(refreshAll(env), /size limit/); + assert.equal(env.objects.size, 0); + assert.equal((await worker.fetch(new Request('https://example.com/sub/nope'), env)).status, 404); + assert.equal((await worker.fetch(new Request('https://example.com/sub/refresh/all', { method: 'POST' }), env)).status, 405); +}); + +test('preserves refresh endpoints, HEAD, ETag and scheduled handler', async t => { + const env = storage(); + t.mock.method(globalThis, 'fetch', async url => upstream(url)); + const refresh = await worker.fetch(new Request('https://example.com/sub/refresh/all'), env); + assert.equal(refresh.status, 200); + assert.equal((await refresh.json()).commit, commit); + const head = await worker.fetch(new Request('https://example.com/sub/clash/ermao.net', { method: 'HEAD' }), env); + assert.equal(await head.text(), ''); + const cached = await worker.fetch(new Request('https://example.com/sub/clash/ermao.net', { headers: { 'If-None-Match': '"etag"' } }), env); + assert.equal(cached.status, 304); + const pending = []; + await worker.scheduled({}, env, { waitUntil(promise) { pending.push(promise); } }); + await Promise.all(pending); + const subscription = await worker.fetch(new Request('https://example.com/sub/refresh/v2ray'), env); + assert.equal(await subscription.text(), 'vmess://test\n'); +}); diff --git a/worker/wrangler.jsonc b/worker/wrangler.jsonc new file mode 100644 index 0000000..faca8d0 --- /dev/null +++ b/worker/wrangler.jsonc @@ -0,0 +1,29 @@ +{ + "name": "sub", + "main": "sub.mjs", + "account_id": "b0021b5742f923f992a9e15ad797de45", + "compatibility_date": "2026-06-03", + "workers_dev": true, + "preview_urls": false, + "routes": [ + { + "pattern": "*.ermao.net/sub/*", + "zone_name": "ermao.net" + } + ], + "triggers": { + "crons": [ + "0 * * * *" + ] + }, + "r2_buckets": [ + { + "binding": "SUBSCRIBE_BUCKET", + "bucket_name": "ermao-net" + } + ], + "observability": { + "enabled": true, + "head_sampling_rate": 0.1 + } +}