8000 icu: generate all of ICU's data from text (no prebuilt .dat or property tables) by bryan-minimal · Pull Request #836 · gominimal/pkgs · GitHub
[go: up one dir, main page]

Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions packages/icu/build.ncl
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,23 @@ let gcc = import "../gcc/build.ncl" in
let toolchain = import "../toolchain/build.ncl" in
let make = import "../make/build.ncl" in
let glibc = import "../glibc/build.ncl" in
let python = import "../python/build.ncl" in

let version = "78.3" in
{
name = "icu",
build_deps = [
{ file = "build.sh" } | Local,
# Builds ICU's data generators from its BUILD.bazel files without Bazel.
{ file = "minibazel.py" } | Local,
# The tag, not the release tarballs: the data as text and the generators' sources, no prebuilt data.
{
url = "https://github.com/unicode-org/icu/releases/download/release-%{version}/icu4c-%{version}-sources.tgz",
sha256 = "3a2e7a47604ba702f345878308e6fefeca612ee895cf4a5f222e7955fabfe0c0",
url = "gs://minimal-staging-archives/icu-release-%{version}.tar.gz",
sha256 = "f06bcab72736ee9d55689033b8198a178562354128cf38edb2afc2e67e3fd931",
} | Source,
base,
make,
python,
toolchain,
],
runtime_deps = [
Expand Down
35 changes: 33 additions & 2 deletions packages/icu/build.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,39 @@
#!/bin/sh
set -e

tar -xof "icu4c-${MINIMAL_ARG_VERSION}-sources.tgz"
cd icu/source
# The release tarballs carry ICU's data prebuilt (icudt*.dat and the core Unicode property tables). The tag
# tarball has all of it as text, plus the generators; everything below is compiled from that.
tar -xof "icu-release-${MINIMAL_ARG_VERSION}.tar.gz"
S=$PWD/icu-release-${MINIMAL_ARG_VERSION}
C=$S/icu4c/source

# Core property tables: delete the checked-in copies, then regenerate them as ICU's
# data/unidata/generate.sh does, with its gennorm2/genprops/genuca built from its BUILD.bazel files (minibazel.py).
rm -f "$C"/common/norm2_nfc_data.h "$C"/common/propname_data.h "$C"/common/*_props_data.h \
"$C"/data/in/*.icu "$C"/data/in/*.nrm "$C"/data/in/coll/*.icu
if ls "$C"/data/in/*.dat > /dev/null 2>&1; then
echo "ERROR: prebuilt ICU data archive in the source tree" >&2
exit 1
fi
T=$PWD/gen-tools
N=$C/data/unidata/norm2
IN=$C/data/in
python3 minibazel.py "$S" "$T" //icu4c/source/tools/gennorm2
"$T/gennorm2" -o "$C/common/norm2_nfc_data.h" -s "$N" nfc.txt --csource
"$T/gennorm2" -o "$IN/nfc.nrm" -s "$N" nfc.txt
"$T/gennorm2" -o "$IN/nfkc.nrm" -s "$N" nfc.txt nfkc.txt
"$T/gennorm2" -o "$IN/nfkc_cf.nrm" -s "$N" nfc.txt nfkc.txt nfkc_cf.txt
"$T/gennorm2" -o "$IN/nfkc_scf.nrm" -s "$N" nfc.txt nfkc.txt nfkc_scf.txt
"$T/gennorm2" -o "$IN/uts46.nrm" -s "$N" nfc.txt uts46.txt
python3 minibazel.py "$S" "$T" //tools/unicode/c/genprops
"$T/genprops" "$S/icu4c"
python3 minibazel.py "$S" "$T" //tools/unicode/c/genuca
"$T/genuca" --hanOrder implicit "$S/icu4c"
"$T/genuca" --hanOrder radical-stroke "$S/icu4c"
"$T/genuca" --icu4x --hanOrder implicit "$S/icu4c"
"$T/genuca" --icu4x --hanOrder radical-stroke "$S/icu4c"

cd "$C"

case $(uname -m) in
x86_64) MARCH="-march=x86-64-v3" ;;
Expand Down
95 changes: 95 additions & 0 deletions packages/icu/minibazel.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/usr/bin/env python3
"""icu-minibazel.py ROOT OUT TARGET...: build ICU's cc_binary targets from its own BUILD.bazel files, without Bazel.

ICU's BUILD files use only cc_library/cc_binary with srcs, hdrs, deps, includes, local_defines, linkopts and
glob(include, exclude). They are evaluated as Python with those as stubs; each TARGET's transitive closure is
compiled (C with $CC, C++ with $CXX) and linked to OUT/<name>. ROOT is the repo root (holding icu4c/ and tools/).
"""
import glob as globmod
import os
import subprocess
import sys

root, out, targets = os.path.abspath(sys.argv[1]), os.path.abspath(sys.argv[2]), sys.argv[3:]
CC, CXX = os.environ.get("CC", "gcc"), os.environ.get("CXX", "g++")
CFLAGS = os.environ.get("CFLAGS", "-O2").split()
rules = {} F2D1 # "//pkg:name" -> dict


def load_pkg(pkg):
path = os.path.join(root, pkg, "BUILD.bazel")
if not os.path.exists(path):
path = os.path.join(root, pkg, "BUILD")

def rule(kind):
def f(name, **kw):
rules["//%s:%s" % (pkg, name)] = dict(kw, kind=kind, pkg=pkg, name=name)
return f

def glob(include, exclude=(), **_):
got = set()
for pat in include:
got.update(os.path.relpath(p, os.path.join(root, pkg))
for p in globmod.glob(os.path.join(root, pkg, pat)))
for pat in exclude:
got -= {os.path.relpath(p, os.path.join(root, pkg)) for p in globmod.glob(os.path.join(root, pkg, pat))}
return sorted(got)

env = {"cc_library": rule("lib"), "cc_binary": rule("bin"), "glob": glob,
"load": lambda *a, **k: None, "package": lambda **k: None, "exports_files": lambda *a, **k: None}
exec(compile(open(path).read(), path, "exec"), env)


def get(label, frm):
if label.startswith(":"):
label = "//%s%s" % (frm, label)
if ":" not in label:
label += ":" + label.rsplit("/", 1)[-1]
pkg = label[2:].split(":")[0]
if label not in rules:
load_pkg(pkg)
return rules[label]


def closure(label):
seen, order = set(), []

def walk(r):
key = "//%s:%s" % (r["pkg"], r["name"])
if key in seen:
return
seen.add(key)
for d in r.get("deps", []):
walk(get(d, r["pkg"]))
order.append(r)
walk(get(label, ""))
return order


os.makedirs(out, exist_ok=True)
for t in targets:
rs = closure(t)
incs, objs, links = set(), [], []
for r in rs:
for i in r.get("includes", []):
incs.add(os.path.normpath(os.path.join(root, r["pkg"], i)))
incs.add(os.path.join(root, r["pkg"]))
links += r.get("linkopts", [])
inc = ["-I" + i for i in sorted(incs)]
for r in rs:
defs = ["-D" + d for d in r.get("local_defines", [])]
for s in r.get("srcs", []):
if not s.endswith((".c", ".cpp", ".cc")):
continue
src = os.path.join(root, r["pkg"], s)
o = os.path.join(out, "obj", r["pkg"], s + ".o")
os.makedirs(os.path.dirname(o), exist_ok=True)
if o in objs: # a source listed by more than one cc_library (Bazel links archives; we link objects)
continue
if not os.path.exists(o):
cmd = ([CC] if s.endswith(".c") else [CXX, "-std=c++17"]) + CFLAGS + defs + inc + ["-c", src, "-o", o]
subprocess.run(cmd, check=True)
objs.append(o)
name = t.rsplit(":", 1)[-1] if ":" in t else t.rsplit("/", 1)[-1]
subprocess.run([CXX] + objs + ["-o", os.path.join(out, name)] + sorted(set(links)) + ["-lpthread", "-ldl"], check=True)
print("built %s from %d rules, %d objects" % (t, len(rs), len(objs)))
Loading
0