diff options
Diffstat (limited to 'maintainers/scripts/update-typst-packages.py')
| -rwxr-xr-x | maintainers/scripts/update-typst-packages.py | 226 |
1 files changed, 226 insertions, 0 deletions
diff --git a/maintainers/scripts/update-typst-packages.py b/maintainers/scripts/update-typst-packages.py new file mode 100755 index 000000000000..345bf2804631 --- /dev/null +++ b/maintainers/scripts/update-typst-packages.py @@ -0,0 +1,226 @@ +#!/usr/bin/env nix-shell +#!nix-shell -p "python3.withPackages (p: with p; [ tomli tomli-w packaging license-expression])" -i python3 + +# This file is formatted with `ruff format`. + +import os +import re +import tomli +import tomli_w +import subprocess +import concurrent.futures +import argparse +import tempfile +import tarfile +from string import punctuation +from packaging.version import Version +from urllib import request +from collections import OrderedDict + + +class TypstPackage: + def __init__(self, **kwargs): + self.pname = kwargs["pname"] + self.version = kwargs["version"] + self.meta = kwargs["meta"] + self.path = kwargs["path"] + self.repo = ( + None + if "repository" not in self.meta["package"] + else self.meta["package"]["repository"] + ) + self.description = self.meta["package"]["description"].rstrip(punctuation) + self.license = self.meta["package"]["license"] + self.params = "" if "params" not in kwargs else kwargs["params"] + self.deps = [] if "deps" not in kwargs else kwargs["deps"] + + @classmethod + def package_name_full(cls, package_name, version): + version_number = map(lambda x: int(x), version.split(".")) + version_nix = "_".join(map(lambda x: str(x), version_number)) + return "_".join((package_name, version_nix)) + + def license_tokens(self): + import license_expression as le + + try: + # FIXME: ad hoc conversion + exception_list = [("EUPL-1.2+", "EUPL-1.2")] + + def sanitize_license_string(license_string, lookups): + if not lookups: + return license_string + return sanitize_license_string( + license_string.replace(lookups[0][0], lookups[0][1]), lookups[1:] + ) + + sanitized = sanitize_license_string(self.license, exception_list) + licensing = le.get_spdx_licensing() + parsed = licensing.parse(sanitized, validate=True) + return [s.key for s in licensing.license_symbols(parsed)] + except le.ExpressionError as e: + print( + f'Failed to parse license string "{self.license}" because of {str(e)}' + ) + exit(1) + + def source(self): + url = f"https://packages.typst.org/preview/{self.pname}-{self.version}.tar.gz" + cmd = [ + "nix", + "store", + "prefetch-file", + "--unpack", + "--hash-type", + "sha256", + "--refresh", + "--extra-experimental-features", + "nix-command", + ] + result = subprocess.run(cmd + [url], capture_output=True, text=True) + hash = re.search(r"hash\s+\'(sha256-.{44})\'", result.stderr).groups()[0] + return url, hash + + def to_name_full(self): + return self.package_name_full(self.pname, self.version) + + def to_attrs(self): + deps = set() + excludes = list(map( + lambda e: os.path.join(self.path, e), + self.meta["package"]["exclude"] if "exclude" in self.meta["package"] else [], + )) + for root, _, files in os.walk(self.path): + for file in filter(lambda f: f.split(".")[-1] == "typ", files): + file_path = os.path.join(root, file) + if file_path in excludes: + continue + with open(file_path, "r") as f: + deps.update( + set( + re.findall( + r"^\s*#import\s+\"@preview/([\w|-]+):(\d+.\d+.\d+)\"", + f.read(), + re.MULTILINE, + ) + ) + ) + self.deps = list( + filter(lambda p: p[0] != self.pname or p[1] != self.version, deps) + ) + source_url, source_hash = self.source() + + return dict( + url=source_url, + hash=source_hash, + typstDeps=[ + self.package_name_full(p, v) + for p, v in sorted(self.deps, key=lambda x: x[0]) + ], + description=self.description, + license=self.license_tokens(), + ) | (dict(homepage=self.repo) if self.repo else dict()) + + +def generate_typst_packages(preview_dir, output_file): + package_tree = dict() + + print("Parsing metadata... from", preview_dir) + for p in os.listdir(preview_dir): + package_dir = os.path.join(preview_dir, p) + for v in os.listdir(package_dir): + package_version_dir = os.path.join(package_dir, v) + with open( + os.path.join(package_version_dir, "typst.toml"), "rb" + ) as meta_file: + try: + package = TypstPackage( + pname=p, + version=v, + meta=tomli.load(meta_file), + path=package_version_dir, + ) + if package.pname in package_tree: + package_tree[package.pname][v] = package + else: + package_tree[package.pname] = dict({v: package}) + except tomli.TOMLDecodeError: + print("Invalid typst.toml:", package_version_dir) + + with open(output_file, "wb") as typst_packages: + + def generate_package(pname, package_subtree): + sorted_keys = sorted(package_subtree.keys(), key=Version, reverse=True) + print(f"Generating metadata for {pname}") + return { + pname: OrderedDict( + (k, package_subtree[k].to_attrs()) for k in sorted_keys + ) + } + + with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor: + sorted_packages = sorted(package_tree.items(), key=lambda x: x[0]) + futures = list() + for pname, psubtree in sorted_packages: + futures.append(executor.submit(generate_package, pname, psubtree)) + packages = OrderedDict( + (package, subtree) + for future in futures + for package, subtree in future.result().items() + ) + print(f"Writing metadata... to {output_file}") + tomli_w.dump(packages, typst_packages) + + +def main(args): + PREVIEW_DIR = "packages/preview" + TYPST_PACKAGE_TARBALL_URL = ( + "https://github.com/typst/packages/archive/refs/heads/main.tar.gz" + ) + + directory = args.directory + if not directory: + tempdir = tempfile.mkdtemp() + print(tempdir) + typst_tarball = os.path.join(tempdir, "main.tar.gz") + + print( + "Downloading Typst packages source from {} to {}".format( + TYPST_PACKAGE_TARBALL_URL, typst_tarball + ) + ) + with request.urlopen( + request.Request(TYPST_PACKAGE_TARBALL_URL), timeout=15.0 + ) as response: + if response.status == 200: + with open(typst_tarball, "wb+") as f: + f.write(response.read()) + else: + print("Download failed") + exit(1) + with tarfile.open(typst_tarball) as tar: + tar.extractall(path=tempdir, filter="data") + directory = os.path.join(tempdir, "packages-main") + directory = os.path.abspath(directory) + + generate_typst_packages( + os.path.join(directory, PREVIEW_DIR), + args.output, + ) + + exit(0) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", "--directory", help="Local Typst Universe repository", default=None + ) + parser.add_argument( + "-o", + "--output", + help="Output file", + default=os.path.join(os.path.abspath("."), "typst-packages-from-universe.toml"), + ) + args = parser.parse_args() + main(args) |