#!/usr/bin/env python3
# Copyright (c) 2026 iispig.com
# SPDX-License-Identifier: GPL-2.0-or-later
#
# This program is free software; you can redistribute it and/or modify it under the terms of
# the GNU General Public License as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version. It is distributed in the hope that it
# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License (GPL-2.0.txt next to
# the published copy of this script) for more details.
"""Subset WenQuanYi Bitmap Song 12/14/16px to the characters the zh pages use.

Run from anywhere with the project venv (fontTools + brotli, pinned in
tools/requirements.txt):

  .venv/bin/python tools/subset-wqy.py OUT_DIR [options]

Input characters
  * every *.html under dist/bbs/ and dist/index.html (default; override with --html GLOB, repeatable):
    text nodes; the attributes alt, placeholder and label (title and aria-label
    are left out: the system draws tooltips in its own font, and nothing draws
    aria-label);
    every <input value>; <meta name="description"> / og:* content; and the
    non-ASCII characters (CSS and JS escapes decoded) of inline <script> and
    <style> bodies and style="" attributes, because a script or a CSS
    content: string can put them on screen;
  * --asset GLOB (repeatable): .js / .css files, scanned the same way as
    inline script/style (default with the default --html: dist/bbs/**/*.js and
    dist/bbs/**/*.css; no match is fine);
  * --text FILE (repeatable): plain-text files, every character counts;
  * --chars STRING: extra characters;
  * REQUIRED below: punctuation/symbols every subset must carry.

Glyph edits (logged in the fonts' name table, the manifest and WQY-NOTICE.txt)
  Outlines are only moved by whole pixels (100 units) and advance widths only
  changed by whole pixels; no pixel is drawn or removed.
  Full-width punctuation goes where XP's SimSun bitmaps had it:
  * WQY12 U+3010 【 sat at the left of its cell like 】; moved right 3 px.
  * ：and ；sat right of centre (Taiwan/Japan style); moved to the left edge:
    12px -4 px, 14px -3 px, 16px -5 px.  WQY12 U+3001 、 moved right 3 px.
  * ？ and ！ were centred; SimSun puts them at the left: ？ 12px -3, 16px -2;
    ！ 12px -1, 14px -2, 16px -2.  WQY12 《 -2 px and 》 +2 px (SimSun's places).
  * WQY14 full-width punctuation advanced 13 px against 14 px for ideographs;
    widened to 14 px (ink stays put) except dashes, ellipsis and box drawing,
    which would break apart.
  Half-width metrics follow the 2008 forum style's "Helvetica, Arial" stack, which drew
  ASCII in Arial on XP (Arial 5.01 hinted widths, hdmx):
  * U+0020 space and U+00A0 no-break space: 3 / 4 / 4 px at 12 / 14 / 16 px
    (WQY: space 6 / 4 / 8, no-break space 7 / 8 / 10).
  * U+007C |: 3 / 4 / 3 px with its 1-px stroke moved to x=1 (WQY 5 / 7 / 4).
  Letters, digits and the hyphen keep WQY's own widths (the hyphen's ink is
  5 px wide at 12px; narrowing it to Arial's 4 px would need a pixel removed).
  Every run checks the result (EXPECT_INK, EXPECT_ADVANCE, bracket weighting)
  and fails with exit status 3 if a glyph is not where it should be.

Dates (GPL-2.0 section 2(a): "the date of any change")
  GLYPH_EDITS_DATE is the date of the edit rules above. The subset date is the
  day the character set was last regenerated: --date YYYY-MM-DD, else
  SOURCE_DATE_EPOCH (build.mjs fills it from the newest dated content entry),
  else the "subsetDate" of the --charset-from manifest, else today (with a
  warning). Both dates go into the name table, the manifest and the notice;
  the same input and date give byte-identical files.

Output (in OUT_DIR)
  wqy12.<hash>.woff2, wqy14.<hash>.woff2, wqy16.<hash>.woff2
      hash = first 8 hex digits of the WOFF2's SHA-256, so a filename changes
      only when the bytes change. Older wqy{size}.<hash>.woff2 files in
      OUT_DIR are removed.
  wqy-manifest.json  per size: file, bytes, glyph/char counts, font metrics,
      the size-adjust that makes 1 font pixel = 1 CSS px, the advance (px) of
      every character that is not full-width (so a build can compute text
      widths exactly), the source TTF and its SHA-256, the glyph edits, the
      character set, and per-page font budgets.
  --css FILE         optional @font-face rules (url prefix: --url-prefix).
  --licence-dir DIR  (default OUT_DIR/../licenses) the files that must ship
      with the fonts under GPL-2.0: GPL-2.0.txt, the upstream README (copyright
      and the font embedding exception; its ISO-8859-1 bytes unchanged, and a
      UTF-8 copy), AUTHORS, the TTF converter's README
      (published as .txt so a server shows it), this script, its requirements,
      a copy of the manifest, a generated, dated WQY-NOTICE.txt (UTF-8 with a
      BOM, so a browser decodes it even when the server sends no charset) and
      a generated index.html that lists every file with local links and shows
      the notice inline (a bare directory is a 403 on nginx). A missing piece
      is an error (exit 1): before writing for the inputs, after writing for
      the bundle itself.
  --source-dir DIR   also copy the input TTFs there (GPL-2.0 section 3: source
      offered from the same place as the fonts). Without it the notice points
      at the upstream release and its SHA-256 only, and a warning is printed.
  --index-extra DIR[=TITLE_EN|TITLE_ZH]  (repeatable) another licence folder of
      the site, with its own index.html (the build passes the Pixel Latin and the
      icon bundles): index.html lists every file in it too, so the URL in the
      fonts' name table (LICENCE_URL) leads to every licence, notice and source
      file of the site. Only index.html changes; the fonts do not.

Rebuilding from the published bundle
  The licence files are looked up under their vendor/ names first and then
  under their published names next to this script, so a copy of the bundle
  plus the three input TTFs is enough:
    python subset-wqy.py OUT_DIR --fonts-dir TTF_DIR --charset-from WQY-manifest.json
  gives byte-identical WOFF2 files (the subset date comes from the manifest).

Checks
  The input TTFs must match the pinned SHA-256 (PINNED_SHA256; --unpinned to
  skip). The font's own metrics are verified: every CJK ideograph must advance
  exactly size*100 units, i.e. 100 units = 1 pixel of the original bitmap.
  A REQUIRED character missing from a font's cmap is an error; so is a page
  character missing from a font, unless listed in --allow-missing (those fall
  back to the next font in the CSS stack). Characters in WARN_CHARS used on a
  page give a warning.

Exit status: 0 ok; 1 bad input/usage/licence files; 2 a character is missing
from a font; 3 a glyph edit did not come out as expected.

--json prints the manifest on stdout (the human-readable report goes to
stderr), which is how build.mjs is meant to call this later.
--project 1000,1500 also measures how big each subset would be with that
many characters (frequency-ranked from --corpus files, topped up with an even
sample of GB2312 level 1) and writes the result into the manifest. It never
writes those projection fonts to disk.
"""
import argparse
import datetime
import glob
import hashlib
import html as htmllib
import io
import json
import os
import platform
import re
import shutil
import sys
import unicodedata
from collections import Counter
from html.parser import HTMLParser

try:
    import fontTools
    from fontTools import subset
    from fontTools.ttLib import TTFont
    import brotli  # (WOFF2 needs it)
except ImportError:  # pragma: no cover
    sys.exit("subset-wqy: fontTools or brotli is missing; run with the project venv: .venv/bin/python tools/subset-wqy.py")

HERE = os.path.dirname(os.path.abspath(__file__))
SITE = os.path.dirname(HERE)
SIZES = (12, 14, 16)
FONT_FILE = "WenQuanYi.Bitmap.Song.{size}px.ttf"
VENDOR_LICENCES = os.path.join(SITE, "vendor", "wqy", "licenses")

# Bump GLYPH_EDITS_DATE whenever the glyph edits or the kind of processing change.
# The date the character set was regenerated is separate (see "Dates" above).
GLYPH_EDITS_DATE = "2026-10-18"
UPSTREAM = {
    "project": "WenQuanYi Bitmap Song 1.0 (Hero) RC1 (0.9.9.8), http://wenq.org/",
    "ttf": "AmusementClub/WenQuanYi-Bitmap-Song-TTF",
    "tag": "v1.0.0-RC1-Build1",
    "url": "https://github.com/AmusementClub/WenQuanYi-Bitmap-Song-TTF/releases/tag/v1.0.0-RC1-Build1",
    "download": "https://github.com/AmusementClub/WenQuanYi-Bitmap-Song-TTF/releases/download/v1.0.0-RC1-Build1/",
}
PINNED_SHA256 = {
    12: "ce94c20796987a169ea08d24207c7a00ff457fe3b5dae5db21c7b51eab60eb3c",
    14: "1da8cb17abb4fd34d4cfc83be7e16baeb49f3822f9236499d29cd9ca7a4f68f6",
    16: "09ebd60aa1900d8d09b61dfc786cbe51454c38b26f0374ead786d7ab96de51be",
}
COPYRIGHT = ("Copyright (c) 2004-2010 The WenQuanYi Project Board of Trustees and Qianqian Fang; "
             "WOFF2 subset (c) 2026 iispig")
LICENCE_URL = "https://iispig.com/fonts/licenses/"

# Characters every subset must carry, whatever the pages say today.
REQUIRED = (
    "".join(chr(c) for c in range(0x20, 0x7F))     # printable ASCII: option (a) Latin
    + "、。，：；！？（）【】「」《》·…—～"            # full-width punctuation used in Chinese text
    + "　"                                       # ideographic space
    + "※◆□★●"                                      # symbols 2008 forum text likes
    + "０１２３４５６７８９"                          # full-width digits
    + "“”‘’"                                         # curly quotes (content uses them)
    + "«»"                                           # pager arrows (WQY's own, whole-pixel, instead of Verdana ‹‹ ››)
    + " "                                       # no-break space (6.0's "date&nbsp;link"), Arial width below
)

# Whitespace the browser draws with the space glyph if the font lacks it
# (HarfBuzz space fallback), so a miss is only a warning.
SOFT_SPACES = {0x00A0, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x202F}

# Characters that render but look wrong; a page using them gets a warning.
WARN_CHARS = {
    0x00B7: "middle dot is half-width and tiny in WQY (2/6/8 px advance, sits low in WQY14); "
            "XP's SimSun had a full-width centred dot. Use the 2008 forum style's separators ' | ', ' - ', '»'",
}

# ---- glyph edits: x offsets in font units (100 units = 1 px), outlines only moved ----
GLYPH_SHIFTS = {
    12: {"【": +300, "：": -400, "；": -400, "、": +300, "？": -300, "！": -100, "《": -200, "》": +200,
         "|": -100},
    14: {"：": -300, "；": -300, "！": -200, "|": -200},
    16: {"：": -500, "；": -500, "？": -200, "！": -200, "|": -100},
}
# Half-width advances set to Arial 5.01's hinted widths at the same pixel size (hdmx),
# because 6.0's "Helvetica, Arial" stack drew ASCII in Arial on XP. Metrics only: the
# space glyphs are empty and the | stroke is moved to x=1 above (Arial's side bearing).
SET_ADVANCE = {
    12: {" ": 300, " ": 300, "|": 300},
    14: {" ": 400, "|": 400},          # the space is already 4 px
    16: {" ": 400, " ": 400, "|": 300},
}
# WQY14: full-width punctuation 1300 -> 1400 units (ideographs are 1400).
WIDEN = {14: (1300, 1400)}
WIDEN_RANGES = ((0x2010, 0x206F), (0x2100, 0x214F), (0x2190, 0x21FF), (0x2200, 0x22FF), (0x2460, 0x24FF),
                (0x25A0, 0x25FF), (0x2600, 0x26FF), (0x3000, 0x303F), (0xFE30, 0xFE4F), (0xFF00, 0xFFEF))
NO_WIDEN = {0x2013, 0x2014, 0x2015, 0x2025, 0x2026, 0x2E3A, 0x2E3B}   # dashes and ellipses must keep joining up
# Ink columns [first, last] (px, inclusive) that must come out of the edits, and the advance (px).
EXPECT_INK = {
    12: {"【": (5, 8, 12), "】": (2, 5, 12), "：": (2, 3, 12), "；": (2, 3, 12), "、": (3, 5, 12),
         "？": (1, 6, 12), "！": (2, 4, 12), "《": (3, 8, 12), "》": (2, 7, 12), "|": (1, 1, 3)},
    14: {"【": (7, 10, 14), "】": (2, 5, 14), "：": (3, 4, 14), "；": (3, 4, 14), "、": (3, 5, 14), "，": (3, 4, 14),
         "。": (3, 6, 14), "？": (3, 8, 14), "！": (3, 5, 14), "|": (1, 1, 4)},
    16: {"【": (9, 13, 16), "】": (3, 7, 16), "：": (2, 3, 16), "；": (2, 3, 16), "、": (3, 6, 16),
         "？": (2, 9, 16), "！": (3, 5, 16), "|": (1, 1, 3)},
}
# Advances (px) of empty glyphs, which EXPECT_INK cannot describe.
EXPECT_ADVANCE = {12: {" ": 3, " ": 3}, 14: {" ": 4, " ": 4}, 16: {" ": 4, " ": 4}}
# Openers must not lean left (SimSun's 12px 《 is centred); closers and ？！ must lean left.
OPENERS, CLOSERS = "【「『《（〔〈“‘", "】」』》）〕〉”’、。，：；？！"

TEXT_ATTRS = {"alt", "placeholder", "label"}   # title / aria-label: drawn by the system or not at all
META_NAMES = {"description"}

PAGE_LOADS = {   # brief 6.3: which WQY files each kind of page loads
    "desktop: index / forum / profile": (12,),
    "desktop: thread": (12, 14),
    "phone: every page": (16, 12),
}
WQY12_BUDGET = 80 * 1024     # brief 6.3: WQY12 subset target; 150 KB = time to slice


class SubsetError(Exception):
    def __init__(self, msg, status):
        super().__init__(msg)
        self.status = status


_CSS_ESC = re.compile(r"\\([0-9a-fA-F]{1,6})[ \t\n\r\f]?")
_JS_ESC = re.compile(r"\\u\{([0-9a-fA-F]{1,6})\}|\\u([0-9a-fA-F]{4})|\\x([0-9a-fA-F]{2})")


def code_chars(s, kind):
    """Non-ASCII characters a script or style sheet may put on screen, escapes decoded."""
    out = [ch for ch in s if ord(ch) > 0x7E]
    pat = _CSS_ESC if kind == "css" else _JS_ESC
    for m in pat.finditer(s):
        h = next(g for g in m.groups() if g)
        cp = int(h, 16)
        if 0x7E < cp <= 0x10FFFF:
            out.append(chr(cp))
    return out


class TextCollector(HTMLParser):
    """Collect the characters a page can show: text, some attributes, inline script/style strings."""

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.chars = Counter()
        self.code = Counter()
        self._raw = []

    def _add(self, s):
        if s:
            self.chars.update(s)

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag in ("script", "style"):
            self._raw.append("css" if tag == "style" else "js")
        for k in TEXT_ATTRS:
            self._add(a.get(k))
        if a.get("style"):
            self.code.update(code_chars(a["style"], "css"))
        if tag == "input":
            self._add(a.get("value"))
        if tag == "meta":
            name = (a.get("name") or a.get("property") or "").lower()
            if name in META_NAMES or name.startswith("og:"):
                self._add(a.get("content"))

    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs)
        if tag in ("script", "style"):
            self._raw.pop()

    def handle_endtag(self, tag):
        if tag in ("script", "style") and self._raw:
            self._raw.pop()

    def handle_data(self, data):
        if self._raw:
            self.code.update(code_chars(data, self._raw[-1]))
        else:
            self._add(data)


def is_ignorable(cp):
    """Code points that never need a glyph: controls, newlines, format characters."""
    if cp < 0x20 or 0x7F <= cp < 0xA0:
        return True
    cat = unicodedata.category(chr(cp))
    return cat in ("Cc", "Cf", "Zl", "Zp")


def collect(html_globs, asset_globs, text_files, extra, recorded=""):
    where = {}          # cp -> first source that used it
    counts = Counter()
    files, assets = [], []
    for pattern in html_globs:
        matched = sorted(glob.glob(pattern, recursive=True))
        if not matched:
            raise SubsetError(f"subset-wqy: no files match {pattern!r}", 1)
        files.extend(matched)
    for pattern in asset_globs:
        assets.extend(sorted(glob.glob(pattern, recursive=True)))

    def note(chars, src):
        for ch, n in chars.items():
            counts[ord(ch)] += n
            where.setdefault(ord(ch), src)

    for path in files:
        p = TextCollector()
        with open(path, encoding="utf-8") as fh:
            p.feed(fh.read())
        p.close()
        rel = os.path.relpath(path, SITE)
        note(p.chars, rel)
        note(p.code, rel + " (script/style)")
    for path in assets:
        with open(path, encoding="utf-8") as fh:
            note(Counter(code_chars(fh.read(), "css" if path.endswith(".css") else "js")), os.path.relpath(path, SITE))
    for path in text_files:
        with open(path, encoding="utf-8") as fh:
            note(Counter(fh.read()), os.path.relpath(path, SITE))
    note(Counter(extra), "--chars")
    note(Counter(recorded), "--charset-from")
    used = {cp for cp in counts if not is_ignorable(cp)}
    return used, where, files, assets


def font_metrics(font, size):
    """Measure the font and check the 100-units-per-pixel grid on CJK advances."""
    upm = font["head"].unitsPerEm
    hhea, os2 = font["hhea"], font["OS/2"]
    cmap = font.getBestCmap()
    hmtx = font["hmtx"]
    px = 100
    bad = [chr(cp) for cp, g in cmap.items() if 0x4E00 <= cp <= 0x9FFF and hmtx[g][0] != size * px]
    if bad:
        raise SubsetError(f"subset-wqy: {size}px font: {len(bad)} CJK glyphs do not advance {size * px} units "
                          f"(e.g. {''.join(bad[:10])}); the 1 px = 100 units assumption is wrong", 1)
    if hhea.ascent % px or hhea.descent % px:
        raise SubsetError(f"subset-wqy: {size}px font: ascent/descent {hhea.ascent}/{hhea.descent} not whole pixels", 1)
    em_px = upm / px                       # CSS px the em must be for 1 font px = 1 CSS px
    content = (hhea.ascent - hhea.descent) // px
    return {
        "unitsPerEm": upm,
        "unitsPerPixel": px,
        "hhea": [hhea.ascent, hhea.descent, hhea.lineGap],
        "typo": [os2.sTypoAscender, os2.sTypoDescender, os2.sTypoLineGap],
        "win": [os2.usWinAscent, os2.usWinDescent],
        "emPx": em_px,
        "sizeAdjust": f"{em_px / size * 100:.6f}".rstrip("0").rstrip(".") + "%",
        "ascentPx": hhea.ascent // px,
        "descentPx": -hhea.descent // px,
        "contentPx": content,
        # (line-height - content) must be even for the baseline to sit on a whole pixel
        "lineHeightParity": "odd" if content % 2 else "even",
    }


def subset_options():
    o = subset.Options()
    o.flavor = "woff2"
    o.hinting = False                 # no instructions (the TTFs have none; this also drops cvt/fpgm/prep/hdmx)
    o.layout_features = []            # no GSUB/GPOS features: no kerning, ligatures or vertical forms wanted
    o.legacy_kern = False
    o.glyph_names = False             # post format 3
    o.notdef_outline = False
    o.recommended_glyphs = False
    o.name_IDs = [0, 1, 2, 3, 4, 5, 6, 8, 9, 11, 12, 13, 14]
    o.name_languages = ["*"]
    o.prune_unicode_ranges = True
    o.prune_codepage_ranges = True
    o.drop_tables += ["DSIG", "vhea", "vmtx", "VDMX", "LTSH", "gasp"]
    o.recalc_timestamp = False        # same input -> same bytes -> same hash
    # fontTools also keeps the bidi mirror of every requested character (≥ pulled in ≤, U+2264, which no
    # page uses): the cmap would then hold a character that is not in the manifest's charset.
    o.bidi_closure = False
    return o


def edit_summary(edits):
    """One line for the name table: what was changed in this file."""
    parts = []
    moved = [e for e in edits if e["edit"] == "move"]
    if moved:
        parts.append("moved " + ", ".join(f"U+{e['cp']:04X} {e['px']:+d}px" for e in moved))
    setadv = [e for e in edits if e["edit"] == "advance"]
    if setadv:
        parts.append("advance " + ", ".join(f"U+{e['cp']:04X} {e['from']}->{e['to']}px" for e in setadv))
    widened = [e for e in edits if e["edit"] == "widen"]
    if widened:
        parts.append(f"{len(widened)} full-width punctuation advances {widened[0]['from']}->{widened[0]['to']}px")
    return "; ".join(parts)


def add_names(font, edits, subset_date):
    note = (f"GNU GPL v2 with font embedding exception, extended to this modified version. Modified by iispig "
            f"with tools/subset-wqy.py (glyph edits {GLYPH_EDITS_DATE}; subset regenerated {subset_date}): subset"
            + (f"; {edit_summary(edits)} (whole pixels, no pixel drawn)" if edits else "")
            + f"; converted to WOFF2. Licence, exception, dated change notice and source: {LICENCE_URL}")
    name = font["name"]
    for plat, enc, lang in ((3, 1, 0x409), (1, 0, 0)):
        name.setName(COPYRIGHT, 0, plat, enc, lang)
        name.setName(note, 13, plat, enc, lang)
        name.setName(LICENCE_URL, 14, plat, enc, lang)


def ink(font, gname):
    """(first ink column, last ink column, advance) in px, or None for an empty glyph."""
    g = font["glyf"][gname]
    adv = font["hmtx"][gname][0]
    if g.numberOfContours == 0:
        return None
    return g.xMin // 100, (g.xMax - 1) // 100, adv // 100


def apply_edits(font, size):
    """Move / widen glyphs in an already-subset font. Returns the list of edits made."""
    glyf, hmtx = font["glyf"], font["hmtx"]
    cmap = font.getBestCmap()
    users = {}
    for cp, g in cmap.items():
        users.setdefault(g, set()).add(cp)
    edits = []

    def own_glyph(ch, what):
        gname = cmap.get(ord(ch))
        if gname is not None and users[gname] != {ord(ch)}:
            raise SubsetError(f"subset-wqy: {size}px: glyph of U+{ord(ch):04X} is shared with "
                              f"{sorted(hex(c) for c in users[gname])}; refusing to {what} it", 3)
        return gname

    for ch, dx in GLYPH_SHIFTS.get(size, {}).items():
        gname = own_glyph(ch, "move")
        if gname is None:
            continue
        g = glyf[gname]
        if g.isComposite() or g.numberOfContours <= 0:
            raise SubsetError(f"subset-wqy: {size}px: {ch} is not a simple outline glyph", 3)
        if dx % 100:
            raise SubsetError(f"subset-wqy: shift for {ch} is not a whole pixel", 3)
        g.coordinates.translate((dx, 0))
        g.recalcBounds(glyf)
        if g.xMin < 0 or g.xMax > hmtx[gname][0]:
            raise SubsetError(f"subset-wqy: {size}px: moving {ch} by {dx // 100}px puts ink outside its cell", 3)
        hmtx[gname] = (hmtx[gname][0], g.xMin)
        edits.append({"edit": "move", "cp": ord(ch), "char": ch, "px": dx // 100})
    for ch, adv in SET_ADVANCE.get(size, {}).items():
        gname = own_glyph(ch, "change the advance of")
        if gname is None:
            continue
        g = glyf[gname]
        if adv % 100:
            raise SubsetError(f"subset-wqy: advance for U+{ord(ch):04X} is not a whole pixel", 3)
        if g.numberOfContours and g.xMax > adv:
            raise SubsetError(f"subset-wqy: {size}px: U+{ord(ch):04X} ink reaches x={g.xMax}, past the new "
                              f"advance {adv}", 3)
        old = hmtx[gname][0]
        if old != adv:
            hmtx[gname] = (adv, hmtx[gname][1])
            edits.append({"edit": "advance", "cp": ord(ch), "char": ch, "from": old // 100, "to": adv // 100})
    if size in WIDEN:
        old, new = WIDEN[size]
        for gname, cps in sorted(users.items(), key=lambda kv: min(kv[1])):
            if hmtx[gname][0] != old:
                continue
            if not all(any(a <= cp <= b for a, b in WIDEN_RANGES) for cp in cps) or cps & NO_WIDEN:
                continue
            g = glyf[gname]
            if g.numberOfContours and g.xMax >= old:      # ink touches the right edge: would open a gap
                continue
            hmtx[gname] = (new, hmtx[gname][1])
            for cp in sorted(cps):
                edits.append({"edit": "widen", "cp": cp, "char": chr(cp), "from": old // 100, "to": new // 100})
    return edits


def check_edits(font, size):
    """Fail loudly if punctuation is not where the edits promise (run on every subset)."""
    cmap = font.getBestCmap()
    problems = []
    for ch, want in EXPECT_INK.get(size, {}).items():
        gname = cmap.get(ord(ch))
        if gname is not None and ink(font, gname) != want:
            problems.append(f"{ch} ink/advance {ink(font, gname)} != {want}")
    for ch, want in EXPECT_ADVANCE.get(size, {}).items():
        gname = cmap.get(ord(ch))
        if gname is not None and font["hmtx"][gname][0] != want * 100:
            problems.append(f"U+{ord(ch):04X} advance {font['hmtx'][gname][0] / 100:g}px != {want}px")
    for chars, side in ((OPENERS, "right"), (CLOSERS, "left")):
        for ch in chars:
            gname = cmap.get(ord(ch))
            if gname is None:
                continue
            first, last, adv = ink(font, gname)
            centre2 = first + last + 1          # 2 x centre of the ink
            if (side == "right" and centre2 < adv) or (side == "left" and centre2 >= adv):
                problems.append(f"{ch} ink {first}-{last} of {adv}px leans the wrong way (want {side} or "
                                f"{'centred' if side == 'right' else 'strictly left'})")
    if problems:
        raise SubsetError(f"subset-wqy: {size}px glyph check failed: " + "; ".join(problems), 3)


def make_subset(path, size, codepoints, subset_date):
    font = TTFont(path, recalcTimestamp=False)
    sub = subset.Subsetter(subset_options())
    sub.populate(unicodes=sorted(codepoints))
    sub.subset(font)
    extra = sorted(set(font.getBestCmap()) - set(codepoints))
    if extra:
        raise SubsetError(f"subset-wqy: {size}px: the subset maps characters that were not asked for: "
                          + ", ".join(f"U+{cp:04X}" for cp in extra), 3)
    edits = apply_edits(font, size)
    check_edits(font, size)
    add_names(font, edits, subset_date)
    font.flavor = "woff2"
    buf = io.BytesIO()
    font.save(buf)
    cmap = font.getBestCmap()
    hmtx = font["hmtx"]
    adv = {chr(cp): hmtx[g][0] // 100 for cp, g in sorted(cmap.items()) if hmtx[g][0] != size * 100}
    return buf.getvalue(), len(font.getGlyphOrder()), len(cmap), edits, adv


def gb2312_level1():
    out = []
    for hi in range(0xB0, 0xD8):
        for lo in range(0xA1, 0xFF):
            if hi == 0xD7 and lo > 0xF9:
                break
            out.append(bytes((hi, lo)).decode("gb2312"))
    return out  # 3755 characters, in GB2312 (pinyin) order


def page_budget(bytes_by_size):
    return {page: sum(bytes_by_size[str(s)] for s in sizes) for page, sizes in PAGE_LOADS.items()
            if all(str(s) in bytes_by_size for s in sizes)}


def project(fonts, base_cps, targets, corpus_files, log, subset_date):
    """Measure subset bytes for bigger character sets instead of guessing."""
    freq = Counter()
    for path in corpus_files:
        with open(path, encoding="utf-8") as fh:
            freq.update(ch for ch in fh.read() if "一" <= ch <= "鿿")
    ranked = [ch for ch, _ in freq.most_common()]
    level1 = gb2312_level1()
    base_cjk = {cp for cp in base_cps if 0x4E00 <= cp <= 0x9FFF}
    base_other = {cp for cp in base_cps if not 0x4E00 <= cp <= 0x9FFF}
    result = {"corpus": [os.path.relpath(p, SITE) for p in corpus_files],
              "corpusDistinctCJK": len(ranked), "samples": []}
    for n in targets:
        # A: the site's characters + the most frequent characters of the corpus,
        #    topped up with an even sample of the rest of GB2312 level 1 if the
        #    corpus runs out (level 1 is in pinyin order, so even = unbiased).
        a = set(base_cjk)
        for ch in ranked:
            if len(a) >= n:
                break
            a.add(ord(ch))
        rest = [ch for ch in level1 if ord(ch) not in a]
        need = n - len(a)
        if need > 0:
            a.update(ord(rest[int(i * len(rest) / need)]) for i in range(need))
        # B: an even sample across all of GB2312 level 1 (the 3755 common characters).
        step = len(level1) / n
        b = {ord(level1[int(i * step)]) for i in range(n)}
        for label, cjk in (("site+corpus", a), ("gb2312-l1-even", b)):
            row = {"sample": label, "cjk": len(cjk), "bytes": {}}
            for size, path in fonts.items():
                data = make_subset(path, size, set(cjk) | base_other | {ord(c) for c in REQUIRED}, subset_date)[0]
                row["bytes"][str(size)] = len(data)
            row["pages"] = page_budget(row["bytes"])
            result["samples"].append(row)
            print(f"  projection {label:15} {len(cjk):5} CJK  " +
                  "  ".join(f"wqy{s} {row['bytes'][str(s)]:>7,} B" for s in fonts) + "  | " +
                  "  ".join(f"{k} {v / 1024:.1f} KB" for k, v in row["pages"].items()), file=log)
    return result


def css_rules(manifest, prefix, unadjusted):
    out = []
    for size, info in manifest["fonts"].items():
        url = f"{prefix}{info['file']}"
        out.append(f'@font-face{{font-family:"{info["family"]}";src:url({url}) format("woff2");'
                   f'size-adjust:{info["metrics"]["sizeAdjust"]};font-display:block}}')
        if unadjusted:
            out.append(f'@font-face{{font-family:"{info["family"]}-noadjust";src:url({url}) format("woff2");'
                       f'font-display:block}}')
    return "\n".join(out) + "\n"


def sha256_file(path):
    h = hashlib.sha256()
    with open(path, "rb") as fh:
        for block in iter(lambda: fh.read(1 << 20), b""):
            h.update(block)
    return h.hexdigest()


# ---- the GPL-2.0 bundle that ships next to the fonts ----
# published name -> (vendor paths to try first, text that proves it is the right file).
# After the vendor paths, the published name next to this script is tried, so the
# published bundle (this script plus its sibling files) can rebuild the fonts by itself.
LICENCE_SOURCES = {
    "GPL-2.0.txt": ([os.path.join(VENDOR_LICENCES, "GPL-2.0.txt")],
                    ("GNU GENERAL PUBLIC LICENSE", "Version 2, June 1991", "END OF TERMS AND CONDITIONS")),
    # the upstream README ("Original Fonts/README" in the TTF repository) is ISO-8859-1: its only
    # non-ASCII byte is 0xA9, the copyright sign. Published twice: the upstream bytes unchanged, and
    # converted to UTF-8 (the copy that is read for the exception text).
    "WQY-README-original.utf8.txt": ([os.path.join(VENDOR_LICENCES, "README-original.utf8.txt")],
                                     ("Qianqian Fang", "As a special exception, if you create a document",
                                      "Copyright: \u00a9 2004-2010")),
    "WQY-README-original.latin1.txt": ([os.path.join(VENDOR_LICENCES, "README-original.latin1.txt")],
                                       ("Qianqian Fang", "As a special exception, if you create a document")),
    "WQY-AUTHORS.txt": ([os.path.join(VENDOR_LICENCES, "AUTHORS.txt")], ("WenQuanYi",)),
    # the converter's Markdown README, copied verbatim; .txt so a static server shows it instead of downloading it
    "WQY-TTF-conversion-README.txt": ([os.path.join(VENDOR_LICENCES, "AmusementClub-README.md")],
                                      ("AmusementClub", "pcf")),
    "subset-wqy.py": ([os.path.abspath(__file__)], ("def make_subset",)),
    "subset-wqy-requirements.txt": ([os.path.join(SITE, "tools", "requirements.txt")], ("fonttools==", "brotli==")),
}
LICENCE_INFO = {   # one line per file for index.html
    "index.html": ("this page", "本页"),
    "WQY-NOTICE.txt": ("dated change notice (GPL-2.0 2(a)), file hashes, how to rebuild, the font embedding exception",
                       "带日期的修改说明、文件哈希、重建方法、字体嵌入例外"),
    "GPL-2.0.txt": ("GNU General Public License version 2, full text", "GPL-2.0 全文"),
    "WQY-README-original.utf8.txt": ("upstream README: copyright and the font embedding exception "
                                     "(converted from ISO-8859-1 to UTF-8)",
                                     "上游说明：版权和字体嵌入例外（从 ISO-8859-1 转成 UTF-8）"),
    "WQY-README-original.latin1.txt": ("the same upstream README, the original bytes (ISO-8859-1)",
                                       "同一份上游说明，原始字节（ISO-8859-1 编码）"),
    "WQY-AUTHORS.txt": ("upstream list of authors", "上游作者名单"),
    "WQY-TTF-conversion-README.txt": ("how AmusementClub made the TTFs from the PCF bitmaps (their Markdown README)",
                                      "AmusementClub 怎么把 PCF 点阵转成 TTF（他们的 Markdown 说明）"),
    "subset-wqy.py": ("the program that made these subsets (part of the source; GPL-2.0-or-later)",
                      "生成这些子集的程序（源码的一部分，GPL-2.0-or-later）"),
    "subset-wqy-requirements.txt": ("package versions the program needs", "程序需要的软件包版本"),
    "WQY-manifest.json": ("character set, hashes, metrics and edits of each file", "每个文件的字表、哈希、度量和修改"),
}
STALE_LICENCE_FILES = ("WQY-TTF-conversion-README.md",)   # earlier name of a bundle file; removed when found
UTF8_BOM = b"\xef\xbb\xbf"


def with_bom(name, data):
    """A .txt or .py file with non-ASCII text gets a UTF-8 BOM (once), so browsers decode it without a charset
    header (Python reads a BOM as the UTF-8 signature, so the published script still runs).
    *.latin1.txt is not UTF-8 and is published byte for byte."""
    if name.endswith(".latin1.txt"):
        return data
    if name.endswith((".txt", ".py")) and not data.startswith(UTF8_BOM) and any(b > 0x7F for b in data):
        return UTF8_BOM + data
    return data


def read_licence_sources():
    out, missing = {}, []
    for name, (paths, marks) in LICENCE_SOURCES.items():
        tried = list(dict.fromkeys(paths + [os.path.join(HERE, name)]))
        data = path = None
        for path in tried:
            try:
                with open(path, "rb") as fh:
                    data = fh.read()
                break
            except OSError:
                continue
        if data is None:
            missing.append(f"{name}: not found (tried {', '.join(tried)})")
            continue
        text = data.decode("utf-8", "replace")
        lacking = [m for m in marks if m not in text]
        if lacking:
            missing.append(f"{name}: {path} does not contain {lacking}")
        out[name] = data
    if not missing:
        lines = out["GPL-2.0.txt"].decode("utf-8").count("\n")
        if lines < 330:
            missing.append(f"GPL-2.0.txt: only {lines} lines; the full text is 339")
    return out, missing


def exception_text(readme_bytes):
    text = readme_bytes.decode("utf-8", "replace")
    m = re.search(r"As a special exception,.*?from your version\.", text, re.S)
    return m.group(0) if m else None


def notice_text(manifest, source_note, source_rel=None):
    fonts = manifest["fonts"]
    edits_date, subset_date = manifest["glyphEditsDate"], manifest["subsetDate"]
    lines = [
        "WenQuanYi Bitmap Song 12px / 14px / 16px: subsets used on iispig.com",
        "=" * 68,
        "",
        "Original font : WenQuanYi Bitmap Song 1.0 (Hero) RC1 (0.9.9.8), http://wenq.org/",
        "Copyright     : (c) 2004-2010 The WenQuanYi Project Board of Trustees and Qianqian Fang",
        "                (see WQY-README-original.utf8.txt and WQY-AUTHORS.txt)",
        f"TrueType      : {UPSTREAM['ttf']}, release {UPSTREAM['tag']}",
        f"                {UPSTREAM['url']}",
        "                (how the TTFs were made from the PCF bitmaps: WQY-TTF-conversion-README.txt)",
        "Licence       : GNU General Public License version 2 (GPL-2.0.txt) with the font",
        "                embedding exception quoted below.",
        "This folder   : index.html lists every file. The upstream files are copied unchanged,",
        "                except that text files with non-ASCII characters get a UTF-8 byte",
        "                order mark in front, so browsers decode them without a charset header.",
        "                The upstream README (\"Original Fonts/README\" in the TrueType repository)",
        "                is ISO-8859-1; its only non-ASCII character is the copyright sign.",
        "                WQY-README-original.latin1.txt is its original bytes, unchanged;",
        "                WQY-README-original.utf8.txt is the same text converted to UTF-8.",
        "",
        "MODIFICATIONS (GPL-2.0 section 2(a))",
        "------------------------------------",
        f"Modified by iispig with subset-wqy.py (a copy is in this folder; it needs the packages",
        f"in subset-wqy-requirements.txt). Generator: fontTools {manifest['fontTools']}, "
        f"brotli {manifest['brotli']}.",
        f"  Glyph edits (items 4 and 5) last changed : {edits_date}",
        f"  These files regenerated (character set)  : {subset_date}",
        "",
        "1. Subset: each file keeps only the characters listed under \"charset\" in",
        "   WQY-manifest.json (the characters the site's pages use, plus a fixed set).",
        "2. Converted from TrueType to WOFF2. Dropped: hinting tables, GSUB/GPOS/kern,",
        "   vhea/vmtx, gasp, VDMX, LTSH, DSIG and glyph names.",
        "3. Name table: copyright (ID 0), this licence note (ID 13) and the licence URL",
        "   (ID 14) added.",
        "4. Punctuation placement. Existing outlines moved by whole pixels, or advance",
        "   widths changed; no pixel was drawn or removed:",
    ]
    latin = []
    for size, info in fonts.items():
        moved = [e for e in info["edits"] if e["edit"] == "move" and e["cp"] > 0x7F]
        widened = [e for e in info["edits"] if e["edit"] == "widen"]
        if not moved and not widened:
            lines.append(f"   - {size}px: none needed for the characters in this subset.")
        for e in moved:
            lines.append(f"   - {size}px: U+{e['cp']:04X} {e['char']} moved {abs(e['px'])} px "
                         f"{'right' if e['px'] > 0 else 'left'}.")
        if widened:
            chars = "".join(e["char"] for e in widened)
            lines.append(f"   - {size}px: advance width {widened[0]['from']} -> {widened[0]['to']} px for "
                         f"{len(widened)} full-width punctuation/symbol characters: {chars}")
        per_cp = {}
        for e in info["edits"]:
            if e["edit"] == "advance" or (e["edit"] == "move" and e["cp"] <= 0x7F):
                per_cp.setdefault(e["cp"], []).append(e)
        latin += [(size, cp, per_cp[cp]) for cp in sorted(per_cp)]
    lines += [
        "5. Half-width metrics set to the widths Arial 5.01 has at the same pixel size",
        "   (the 2008 forum style this site imitates used \"Helvetica, Arial\", which drew ASCII",
        "   in Arial on Windows XP).",
        "   Advance widths only, plus the | stroke moved by whole pixels; no pixel drawn:",
    ]
    if not latin:
        lines.append("   - none needed for the characters in this subset.")
    for size, cp, es in latin:
        what = []
        for e in es:
            if e["edit"] == "move":
                what.append(f"stroke moved {abs(e['px'])} px {'right' if e['px'] > 0 else 'left'}")
            else:
                what.append(f"advance {e['from']} -> {e['to']} px")
        lines.append(f"   - {size}px: U+{cp:04X} {unicodedata.name(chr(cp)).lower()}: {', '.join(what)}.")
    lines += [
        "",
        "FILES",
        "-----",
    ]
    for size, info in fonts.items():
        src = info["source"]
        lines.append(f"{info['file']}  sha256 {info['sha256']}")
        lines.append(f"    made from {src['file']}  sha256 {src['sha256']}")
    lines += [
        "",
        "SOURCE CODE (GPL-2.0 section 3)",
        "-------------------------------",
        "The source of these files is the input TTF named above for each file, plus",
        "subset-wqy.py and the character list in WQY-manifest.json. subset-wqy.py is",
        "Copyright (c) 2026 iispig.com, licensed under the GNU General Public License",
        "version 2 or (at your option) any later version (GPL-2.0.txt).",
    ] + ([
        f"To rebuild the files, copy this folder with {source_rel}/, install the package",
        "versions in subset-wqy-requirements.txt and run, in the copy:",
        f"  python subset-wqy.py OUT_DIR --fonts-dir {source_rel} --charset-from WQY-manifest.json",
    ] if source_rel else [
        "To rebuild the files, copy this folder, put the three input TTFs in a folder",
        "TTF_DIR, install the package versions in subset-wqy-requirements.txt and run,",
        "in the copy:",
        "  python subset-wqy.py OUT_DIR --fonts-dir TTF_DIR --charset-from WQY-manifest.json",
    ]) + [
        "subset-wqy.py finds the other files of this folder next to itself, and takes the",
        f"subset date ({subset_date}) from WQY-manifest.json, so the WOFF2 files come out",
        "byte for byte the same. (Inside the iispig site tree the same command works as",
        "  python tools/subset-wqy.py OUT_DIR --charset-from WQY-manifest.json",
        "with the TTFs in vendor/wqy/.)",
    ] + source_note + [
        "",
        "FONT EMBEDDING EXCEPTION",
        "------------------------",
        "As allowed by its last sentences, iispig extends this exception to the",
        "modified versions above. The exception, as written by the WenQuanYi Project:",
        "",
    ] + ["    " + ln for ln in manifest["_exception"].splitlines()] + [
        "",
        "中文摘要：以上字体是文泉驿点阵宋体（GPL-2.0，附字体嵌入例外）的子集，由 iispig 修改：子集化；"
        "个别全角标点整像素平移；半角空格、不换行空格和竖线的步进改成同字号 Arial 的宽度（竖线整像素左移）；"
        "WQY14 全角标点步进从 13px 改成 14px；转成 WOFF2。没有画或删任何像素。"
        f"修改规则定于 {edits_date}，本次子集生成于 {subset_date}。",
        "",
    ]
    return "\n".join(lines)


def extra_listing(specs, licence_dir):
    """--index-extra DIR[=TITLE_EN|TITLE_ZH]: other licence folders of the site, listed on index.html."""
    out = []
    for spec in specs:
        d, _, title = spec.partition("=")
        en, _, zh = title.partition("|")
        d = os.path.abspath(d)
        if not os.path.isfile(os.path.join(d, "index.html")):
            raise SubsetError(f"subset-wqy: --index-extra {d}: no index.html there (a bare folder is a 403 on nginx)", 1)
        rel = os.path.relpath(d, licence_dir).replace(os.sep, "/")
        files = []
        for root, dirs, names in os.walk(d):
            dirs.sort()
            for n in sorted(names):
                if n.startswith("."):
                    continue
                full = os.path.join(root, n)
                files.append((os.path.relpath(full, d).replace(os.sep, "/"), os.path.getsize(full)))
        files.sort(key=lambda f: (f[0] != "index.html", f[0].count("/"), f[0]))
        out.append({"rel": rel, "en": en or rel, "zh": zh, "files": files})
    return out


def licence_index(manifest, bundle, notice, fonts_rel, source_rel, extras=()):
    """index.html for the licence folder: local links only, deterministic."""
    esc = htmllib.escape
    rows = []

    def row(href, size, en, zh):
        rows.append(f'<tr><td><a href="{esc(href)}">{esc(href)}</a></td><td class="n">{size:,}</td>'
                    f'<td>{esc(en)}<br>{esc(zh)}</td></tr>')
    for name in ["WQY-NOTICE.txt"] + sorted(n for n in bundle if n != "WQY-NOTICE.txt"):
        row(name, len(bundle[name]), *LICENCE_INFO[name])
    for size, info in manifest["fonts"].items():
        row(f"{fonts_rel}/{info['file']}", info["bytes"],
            f"WQY{size}: the font itself (WOFF2), made from {info['source']['file']}",
            f"WQY{size} 字体文件本身（WOFF2）")
    if source_rel:
        for size, info in manifest["fonts"].items():
            row(f"{source_rel}/{info['source']['file']}", info["source"]["bytes"],
                f"source: the input TTF of WQY{size}, unchanged (GPL-2.0 section 3; SHA-256 {info['source']['sha256'][:16]}...)",
                f"源文件：WQY{size} 的输入 TTF，未修改（GPL-2.0 第 3 条）")
    more = []
    if extras:
        more += ["<h2>Other licence folders on this site 本站其他许可目录</h2>",
                 "<p>The other fonts and the icons of the site come with their own licences, notices and (for the "
                 "fonts) source files; every file of those folders is listed here too.</p>",
                 "<p>本站的其他字体和图标有各自的许可和说明（字体还有源文件），这些目录里的每个文件也列在下面。</p>",
                 "<table>",
                 "<tr><th>File 文件</th><th>Bytes 字节</th><th>Contents 内容</th></tr>"]
        for x in extras:
            more.append(f'<tr><th colspan="3"><a href="{esc(x["rel"])}/index.html">{esc(x["rel"])}/</a>: '
                        f'{esc(x["en"])}{"<br>" + esc(x["zh"]) if x["zh"] else ""}</th></tr>')
            for name, size in x["files"]:
                href = f'{x["rel"]}/{name}'
                desc = ("that folder's list of files and its notice", "该目录的文件列表和说明") if name == "index.html" else ("", "")
                more.append(f'<tr><td><a href="{esc(href)}">{esc(href)}</a></td><td class="n">{size:,}</td>'
                            f'<td>{esc(desc[0])}{"<br>" + esc(desc[1]) if desc[1] else ""}</td></tr>')
        more.append("</table>")
    return "\n".join([
        "<!doctype html>",
        '<html lang="zh-CN">',
        "<head>",
        '<meta charset="utf-8">',
        '<meta name="viewport" content="width=device-width, initial-scale=1">',
        "<title>文泉驿点阵宋体子集：许可和来源</title>",
        "<style>body{margin:16px;max-width:960px;font:14px/1.5 sans-serif;color:#000;background:#fff}"
        "table{border-collapse:collapse;margin:0 0 16px}th,td{border:1px solid #ccc;padding:2px 6px;"
        "text-align:left;vertical-align:top}td.n{text-align:right;white-space:nowrap}"
        "pre{white-space:pre-wrap;word-break:break-word;font:13px/1.45 monospace;background:#f7f7f7;"
        "border:1px solid #ccc;padding:8px}</style>",
        "</head>",
        "<body>",
        "<h1>WenQuanYi Bitmap Song subsets on iispig.com: licence and source</h1>",
        "<p>The WQY12 / WQY14 / WQY16 web fonts on this site are modified versions of WenQuanYi Bitmap Song, "
        "licensed under the GNU GPL version 2 with the font embedding exception. Every file that goes with "
        "them is listed below; the dated change notice follows in full.</p>",
        "<p>本站的 WQY12 / WQY14 / WQY16 网页字体是文泉驿点阵宋体的修改版，许可是 GPL-2.0 附字体嵌入例外。"
        "下面列出随字体发布的全部文件，再往下是带日期的修改说明全文。</p>",
        "<table>",
        "<tr><th>File 文件</th><th>Bytes 字节</th><th>Contents 内容</th></tr>",
        *rows,
        "</table>",
        *more,
        "<h2>WQY-NOTICE.txt</h2>",
        f"<pre>{esc(notice)}</pre>",
        "</body>",
        "</html>",
        "",
    ])


def _same_file(path, data):
    try:
        with open(path, "rb") as fh:
            return fh.read() == data
    except OSError:
        return False


def pick_subset_date(args, from_manifest):
    """The date the character set was regenerated (see "Dates" in the module docstring)."""
    if args.date:
        d, why = args.date, "--date"
    elif os.environ.get("SOURCE_DATE_EPOCH"):
        try:
            d = datetime.datetime.fromtimestamp(int(os.environ["SOURCE_DATE_EPOCH"]),
                                                datetime.timezone.utc).date().isoformat()
        except (ValueError, OverflowError, OSError):
            raise SubsetError(f"subset-wqy: SOURCE_DATE_EPOCH={os.environ['SOURCE_DATE_EPOCH']!r} is not a Unix time", 1)
        why = "SOURCE_DATE_EPOCH"
    elif from_manifest and (from_manifest.get("subsetDate") or from_manifest.get("modified")):
        d, why = from_manifest.get("subsetDate") or from_manifest["modified"], "--charset-from manifest"
    else:
        d, why = datetime.date.today().isoformat(), "today"
        print(f"warn  no --date or SOURCE_DATE_EPOCH: the change notice dates this subset {d}; pass --date "
              "(or SOURCE_DATE_EPOCH) for files that do not change with the calendar", file=sys.stderr)
    try:
        datetime.date.fromisoformat(d)
    except ValueError:
        raise SubsetError(f"subset-wqy: subset date {d!r} ({why}) is not YYYY-MM-DD", 1)
    if len(d) != 10:
        raise SubsetError(f"subset-wqy: subset date {d!r} ({why}) is not YYYY-MM-DD", 1)
    # a subset cannot predate the edit rules it was made with
    return max(d, GLYPH_EDITS_DATE)


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    ap.add_argument("out_dir")
    ap.add_argument("--html", action="append", help="glob of HTML files (default dist/bbs/**/*.html and dist/index.html)")
    ap.add_argument("--asset", action="append", help="glob of .js/.css files whose strings may be shown "
                    "(default with the default --html: dist/bbs/**/*.js and *.css)")
    ap.add_argument("--text", action="append", default=[], help="plain-text file whose characters are all included")
    ap.add_argument("--chars", default="", help="extra characters")
    ap.add_argument("--charset-from", metavar="MANIFEST",
                    help="rebuild from the character sets recorded in a wqy-manifest.json (no HTML is read)")
    ap.add_argument("--fonts-dir", default=os.path.join(SITE, "vendor", "wqy"))
    ap.add_argument("--unpinned", action="store_true", help="accept input TTFs whose SHA-256 differs from the pinned one")
    ap.add_argument("--sizes", default="12,14,16")
    ap.add_argument("--allow-missing", default="", help="page characters that may fall back to another font")
    ap.add_argument("--css", help="also write @font-face rules to this file")
    ap.add_argument("--url-prefix", default="", help="prefix for font URLs in --css (e.g. /fonts/zh/)")
    ap.add_argument("--css-unadjusted", action="store_true",
                    help="also emit <family>-noadjust faces without size-adjust (for comparison specimens)")
    ap.add_argument("--licence-dir", help="where the GPL bundle goes (default OUT_DIR/../licenses)")
    ap.add_argument("--source-dir", help="also copy the input TTFs here (GPL-2.0 section 3)")
    ap.add_argument("--index-extra", action="append", default=[], metavar="DIR[=TITLE_EN|TITLE_ZH]",
                    help="another licence folder of the site (with its own index.html): index.html also lists "
                         "its files with relative links, so the URL in the fonts' name table leads to every "
                         "licence, notice and source file of the site (repeatable)")
    ap.add_argument("--date", help="subset date YYYY-MM-DD for the change notice (default: SOURCE_DATE_EPOCH, "
                    "else the --charset-from manifest's subsetDate, else today)")
    ap.add_argument("--json", action="store_true", help="print the manifest JSON on stdout")
    ap.add_argument("--project", default="", help="comma-separated character counts to project, e.g. 1000,1500")
    ap.add_argument("--corpus", action="append", default=[], help="text file(s) ranking characters for --project")
    args = ap.parse_args(argv)
    try:
        return run(ap, args)
    except SubsetError as e:
        print(e, file=sys.stderr)
        return e.status


def run(ap, args):
    log = sys.stderr if args.json else sys.stdout

    try:
        sizes = [int(s) for s in args.sizes.split(",") if s]
    except ValueError:
        ap.error("--sizes must be like 12,14,16")
    if any(s not in SIZES for s in sizes):
        ap.error(f"--sizes must be from {SIZES}")
    fonts = {s: os.path.join(args.fonts_dir, FONT_FILE.format(size=s)) for s in sizes}
    source_hash = {}
    for s, p in fonts.items():
        if not os.path.isfile(p):
            raise SubsetError(f"subset-wqy: missing font {p}", 1)
        source_hash[s] = sha256_file(p)
        if source_hash[s] != PINNED_SHA256[s] and not args.unpinned:
            raise SubsetError(f"subset-wqy: {p} has SHA-256 {source_hash[s]}, not the pinned {PINNED_SHA256[s]} "
                              f"({UPSTREAM['tag']}); pass --unpinned to use it anyway", 1)

    out_dir = os.path.abspath(args.out_dir)
    licence_dir = os.path.abspath(args.licence_dir or os.path.join(out_dir, "..", "licenses"))
    licence_files, missing = read_licence_sources()
    if missing:
        for m in missing:
            print(f"ERROR licence bundle: {m}", file=sys.stderr)
        raise SubsetError("subset-wqy: FAILED: the GPL-2.0 bundle is incomplete; nothing written", 1)
    exception = exception_text(licence_files["WQY-README-original.utf8.txt"])
    if not exception:
        raise SubsetError("subset-wqy: font embedding exception not found in the upstream README", 1)

    extra, recorded = args.chars, ""
    from_manifest = None
    if args.charset_from:
        with open(args.charset_from, encoding="utf-8") as fh:
            from_manifest = json.load(fh)
        recorded = "".join(f["charset"] for f in from_manifest["fonts"].values())
    subset_date = pick_subset_date(args, from_manifest)
    default_inputs = not args.html and not args.charset_from
    html_globs = args.html or ([] if args.charset_from else [os.path.join(SITE, "dist", "bbs", "**", "*.html"),
                                                             os.path.join(SITE, "dist", "index.html")])
    asset_globs = args.asset or ([os.path.join(SITE, "dist", "bbs", "**", "*.js"),
                                  os.path.join(SITE, "dist", "bbs", "**", "*.css")] if default_inputs else [])
    used, where, files, assets = collect(html_globs, asset_globs, args.text, extra, recorded)
    required = {ord(c) for c in REQUIRED}
    allow = {ord(c) for c in args.allow_missing}
    wanted = used | required
    print(f"subset-wqy: {len(files)} HTML file(s), {len(assets)} asset file(s), {len(args.text)} text file(s); "
          f"{len(used)} characters used, {len(wanted)} with the required set", file=log)
    warnings = []
    for cp in sorted(used & set(WARN_CHARS)):
        if where.get(cp) == "--charset-from":      # a recorded character set, not a page using it
            continue
        msg = f"U+{cp:04X} {chr(cp)!r} ({where.get(cp)}): {WARN_CHARS[cp]}"
        warnings.append(msg)
        print(f"warn  {msg}", file=sys.stderr)

    manifest = {"generator": "tools/subset-wqy.py", "glyphEditsDate": GLYPH_EDITS_DATE, "subsetDate": subset_date,
                "fontTools": fontTools.version,
                "brotli": getattr(brotli, "__version__", "?"), "python": platform.python_version(),
                "upstream": UPSTREAM, "required": REQUIRED,
                "inputs": {"html": len(files), "assets": len(assets), "text": len(args.text),
                           "used": len(used), "wanted": len(wanted)},
                "fonts": {}}
    # check every font before writing anything, so a failure leaves OUT_DIR as it was
    failed, checked = False, {}
    for size, path in fonts.items():
        font = TTFont(path, lazy=True)
        metrics = font_metrics(font, size)
        cmap = font.getBestCmap()
        font.close()
        miss_req = sorted(cp for cp in required if cp not in cmap)
        miss_used = sorted(cp for cp in used - required if cp not in cmap)
        hard = [cp for cp in miss_used if cp not in allow and cp not in SOFT_SPACES]
        soft = [cp for cp in miss_used if cp in allow or cp in SOFT_SPACES]
        for cp in miss_req:
            print(f"ERROR wqy{size}: required U+{cp:04X} {chr(cp)!r} is not in the font", file=sys.stderr)
        for cp in hard:
            print(f"ERROR wqy{size}: U+{cp:04X} {chr(cp)!r} ({where.get(cp)}) is not in the font", file=sys.stderr)
        for cp in soft:
            print(f"warn  wqy{size}: U+{cp:04X} {chr(cp)!r} ({where.get(cp)}) not in the font; falls back", file=sys.stderr)
        failed = failed or bool(miss_req or hard)
        checked[size] = (metrics, {cp for cp in wanted if cp in cmap}, soft)
    if failed:
        raise SubsetError("subset-wqy: FAILED: characters missing from the font(s) above; nothing written. Rewrite "
                          "the text, or pass them in --allow-missing if a fallback font is intended.", 2)

    # build every font in memory first (the glyph checks can still fail)
    built = {}
    for size, path in fonts.items():
        metrics, present, soft = checked[size]
        built[size] = make_subset(path, size, present, subset_date)

    os.makedirs(out_dir, exist_ok=True)
    for size, path in fonts.items():
        metrics, present, soft = checked[size]
        data, glyphs, cmap_count, edits, adv = built[size]
        digest = hashlib.sha256(data).hexdigest()
        name = f"wqy{size}.{digest[:8]}.woff2"
        target = os.path.join(out_dir, name)
        if not os.path.exists(target):
            with open(target, "wb") as fh:
                fh.write(data)
        for old in os.listdir(out_dir):
            if re.fullmatch(rf"wqy{size}\.[0-9a-f]{{8}}\.woff2", old) and old != name:
                os.remove(os.path.join(out_dir, old))
                print(f"  removed stale {old}", file=log)
        chars = "".join(chr(cp) for cp in sorted(present))
        manifest["fonts"][str(size)] = {
            "family": f"WQY{size}", "file": name, "bytes": len(data), "sha256": digest,
            "source": {"file": os.path.basename(path), "bytes": os.path.getsize(path), "sha256": source_hash[size],
                       "url": UPSTREAM["download"] + os.path.basename(path)},
            "glyphs": glyphs, "chars": cmap_count,
            "cjk": sum(1 for cp in present if 0x4E00 <= cp <= 0x9FFF),
            "fellBack": "".join(chr(cp) for cp in soft),
            "metrics": metrics,
            "edits": edits,
            "advancePx": {"default": size, "other": adv},
            "charset": chars,
            "charsetSha256": hashlib.sha256(chars.encode("utf-8")).hexdigest(),
        }
        moved = " ".join(f"{e['char']}{e['px']:+d}" for e in edits if e["edit"] == "move")
        setadv = " ".join(f"U+{e['cp']:04X} {e['from']}->{e['to']}" for e in edits if e["edit"] == "advance")
        nwide = sum(1 for e in edits if e["edit"] == "widen")
        print(f"  wqy{size}  {cmap_count:5} chars  {glyphs:5} glyphs  {len(data):>8,} bytes  {name}  "
              f"(size-adjust {metrics['sizeAdjust']}, content {metrics['contentPx']}px, "
              f"{metrics['lineHeightParity']} line-heights; moved {moved or '-'}; advance {setadv or '-'}"
              f"{f'; {nwide} advances widened' if nwide else ''})", file=log)
    by_size = {s: f["bytes"] for s, f in manifest["fonts"].items()}
    manifest["budget"] = {"wqy12MaxBytes": WQY12_BUDGET, "loads": {k: list(v) for k, v in PAGE_LOADS.items()},
                          "pages": page_budget(by_size)}
    print("  per page: " + "  ".join(f"{k} {v / 1024:.1f} KB" for k, v in manifest["budget"]["pages"].items()), file=log)
    if "12" in by_size and by_size["12"] > WQY12_BUDGET:
        warnings.append(f"WQY12 is {by_size['12']:,} B, over the {WQY12_BUDGET:,} B target")
        print(f"warn  {warnings[-1]}", file=sys.stderr)
    if warnings:
        manifest["warnings"] = warnings
    if args.project:
        targets = [int(n) for n in args.project.split(",") if n]
        manifest["projection"] = project(fonts, used | required, targets, args.corpus, log, subset_date)

    # ---- the licence bundle ----
    source_note, source_rel = [], None
    if args.source_dir:
        src_dir = os.path.abspath(args.source_dir)
        os.makedirs(src_dir, exist_ok=True)
        for size, path in fonts.items():
            dst = os.path.join(src_dir, os.path.basename(path))
            if not (os.path.exists(dst) and sha256_file(dst) == source_hash[size]):
                shutil.copyfile(path, dst)
        rel = source_rel = os.path.relpath(src_dir, licence_dir).replace(os.sep, "/")
        source_note = [f"The input TTFs are published unchanged in {rel}/ next to this notice, from the",
                       "same place as the fonts (the upstream release files, SHA-256 above)."]
    else:
        source_note = ["The input TTFs are the upstream release files (SHA-256 above):",
                       f"  {UPSTREAM['download']}<file name>"]
        print("warn  no --source-dir: the notice only points at the upstream release. For a public "
              "deployment, host the input TTFs next to the fonts (--source-dir) or add a written offer "
              "(GPL-2.0 section 3).", file=sys.stderr)
    manifest["licences"] = {"dir": os.path.relpath(licence_dir, out_dir).replace(os.sep, "/"),
                            "files": sorted([*licence_files, "WQY-NOTICE.txt", "WQY-manifest.json", "index.html"]),
                            "url": LICENCE_URL, "sourceBundled": bool(args.source_dir)}
    manifest_bytes = (json.dumps(manifest, ensure_ascii=False, indent=1) + "\n").encode("utf-8")
    manifest["_exception"] = exception
    notice_str = notice_text(manifest, source_note, source_rel)
    del manifest["_exception"]
    os.makedirs(licence_dir, exist_ok=True)
    bundle = {n: with_bom(n, d) for n, d in licence_files.items()}
    # BOM: a static server without "charset utf-8" sends text/plain with no charset, and browsers then
    # guess the encoding wrong; with a BOM they decode UTF-8 whatever the header says
    bundle["WQY-NOTICE.txt"] = UTF8_BOM + notice_str.encode("utf-8")
    bundle["WQY-manifest.json"] = manifest_bytes
    fonts_rel = os.path.relpath(out_dir, licence_dir).replace(os.sep, "/")
    bundle["index.html"] = licence_index(manifest, bundle, notice_str, fonts_rel, source_rel,
                                         extra_listing(args.index_extra, licence_dir)).encode("utf-8")
    for old in STALE_LICENCE_FILES:
        if old not in bundle and os.path.isfile(os.path.join(licence_dir, old)):
            os.remove(os.path.join(licence_dir, old))
            print(f"  removed stale licences/{old}", file=log)
    for name, data in bundle.items():
        target = os.path.join(licence_dir, name)
        try:
            with open(target, "rb") as fh:
                if fh.read() == data:
                    continue
        except OSError:
            pass
        with open(target, "wb") as fh:
            fh.write(data)
    # every file the notice and index.html promise must now be there, byte for byte
    absent = [n for n, d in bundle.items() if not _same_file(os.path.join(licence_dir, n), d)]
    if source_rel:
        absent += [f"{source_rel}/{f['source']['file']}" for f in manifest["fonts"].values()
                   if not os.path.isfile(os.path.join(licence_dir, source_rel, f["source"]["file"]))]
    absent += [f"{fonts_rel}/{f['file']}" for f in manifest["fonts"].values()
               if not os.path.isfile(os.path.join(out_dir, f["file"]))]
    if absent:
        raise SubsetError(f"subset-wqy: FAILED: licence bundle incomplete after writing: {absent}", 1)
    print(f"  licence bundle: {len(bundle)} files in {os.path.relpath(licence_dir, SITE)} "
          f"(glyph edits {GLYPH_EDITS_DATE}, subset {subset_date})", file=log)

    with open(os.path.join(out_dir, "wqy-manifest.json"), "wb") as fh:
        fh.write(manifest_bytes)
    if args.css:
        with open(args.css, "w", encoding="utf-8") as fh:
            fh.write("/* generated by tools/subset-wqy.py -- do not edit */\n")
            fh.write(css_rules(manifest, args.url_prefix, args.css_unadjusted))
    if args.json:
        sys.stdout.write(manifest_bytes.decode("utf-8"))
    return 0


if __name__ == "__main__":
    sys.exit(main())