#!/usr/bin/env python3 """Builds the German capitalization sidecar (HEXKeyboard/TypingModel/de.capitals) for the lowercase-only German typing model (de.hextm), from the same Google Books Ngram v3 German unigram shards (2020-02-17, CC BY 3.0) that model was counted from. python3 scripts/build-german-capitals.py HEXKeyboard/TypingModel/de.hextm \ HEXKeyboard/TypingModel/de.capitals For every word in the model it sums the 1980-2019 counts of that exact all-lowercase token and of the token with only its first letter uppercased (ä, ö and ü included; a word that begins with ß has no capital spelling, and ß anywhere else stays as it is). Other casings (ALLCAPS, mixed) and POS-tagged tokens (any with "_") are ignored. A word is written when the capitalized share rounds to at least SOMETIMES (50%) and the two counts together reach MIN_TOTAL (40 occurrences across 40 years of books, below which the share is mostly noise), except pronouns and possessives that are capitalized for other reasons (FUNCTION_WORDS). The counts decide every percentage; the caps below are applied after them, and the merged counts are kept, so a rule can change without streaming the shards again. Three kinds of word are capped at CAP (94%) so the keyboard only capitalizes them after an article or determiner, because they are also common lowercase words: - a verb form: a word ending in "e" whose word+"n" is its infinitive, where both readings are used enough to type. The infinitive's own capitalised share can't tell them apart ("Fragen" the plural outnumbers "fragen" the verb, 74%), so the rule reads the counts instead: the infinitive needs VERB_MIN_LOW lowercase uses, the -e word at least VERB_MIN_SHARE of its own uses lowercase (frage 5.4%, reise 2.7%), and the infinitive between VERB_MIN_RATIO and VERB_MAX_RATIO times the lowercase -e word, which is the band a conjugation fills (frage 2.8x, drucke 23x). A noun without a verb of its own is outside it: "hause" is 0.6% lowercase, and "wegen" and "lagen" are 250x and 43x the lowercase "wege" and "lage", so "haus", "hause", "wege" and "lage" keep their capitals. - a stem: the same test for a consonant stem and its word+"en" ("kauf"/"kaufen", "schlaf"/ "schlafen", "sitz"/"sitzen"), the imperative and casual first person people type. Most nouns are built from a verb stem too, so its band is wider (STEM_*): "kaufen" may be 38x the lowercase "kauf". A noun the books nearly always capitalize stays: "bild" is 0.6% lowercase, "bericht" 1.1%, "tisch" has "tischen" at only 0.9x it. - ALSO_LOWERCASE: words whose lowercase use is common but no rule above sees it, and every two-letter word, which is a chat abbreviation ("gl", "rn"), never a noun. Each line is `wordpercent`, sorted by word, with a trailing newline. The eight shards are streamed through curl and gzip -dc, four at a time, and never stored. Each shard's counts are cached under .build-german-capitals/ beside this script, so an interrupted run resumes where it stopped. Their merged per-token counts stay there as counts.tsv after the run, so a later rule change renders in seconds. """ import hashlib, multiprocessing, os, struct, subprocess, sys BASE = "https://storage.googleapis.com/books/ngrams/books/20200217" FILES = ("ger", 8) # ger/1-0000N-of-00008.gz, the German unigram shards FIRST_YEAR, LAST_YEAR = 1980, 2019 MIN_TOTAL = 40 # occurrences (lowercase + capitalized) a word needs SOMETIMES = 50 # the share from which a word is written to the list CAP = 94 # a capped word is below the keyboard's 95% "always" share VERB_MIN_LOW = 200_000 # lowercase uses of the infinitive over 40 years, 5,000 a year VERB_MIN_SHARE = 14 # in tenths of a percent: 1.4% of the word's uses lowercase VERB_MIN_RATIO = 2 # the infinitive at least twice the lowercase -e form VERB_MAX_RATIO = 24 # and at most 24 times it STEM_MIN_LOW = 140_000 # a consonant infinitive's lowercase uses, 3,500 a year STEM_MIN_SHARE = 13 # in tenths of a percent: 1.3% of the stem's uses lowercase STEM_MIN_RATIO = 2 # the infinitive at least twice the lowercase stem STEM_MAX_RATIO = 60 # and at most 60 times it: the imperative is rarer than "ich kaufe" SHARDS = 4 # shards streamed at once WORK = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".build-german-capitals") FUNCTION_WORDS = frozenset(""" sie ihr ihre ihren ihrem ihrer ihres ihnen du dich dir dein deine deinen deinem deiner deines euch euer eure euren eurem eurer eures ich mich mir wir uns er es ihn ihm man sich mein meine meinen meinem meiner meines """.split()) # Words the books capitalize nearly always whose lowercase use is still common, and that # the two rules above cannot see: # - "teil" and "preis" stand alone as the separable prefixes of "teilnehmen" and # "preisgeben" ("ich nehme teil", "er gibt es preis"); # - "klasse" is the adjective and adverb before "Klasse" ("das ist klasse"); # - filme, rolle, presse, pumpe, campe, pflanze, beichte, fische and taufe are first person and # imperative too ("ich filme", "roll!"), but their infinitives are too rare or their -e # word is too rarely lowercase for the verb rule. "erbe" stays out: people type "das Erbe" # and "der Erbe". ALSO_LOWERCASE = frozenset(""" beichte campe filme fische klasse pflanze presse preis pumpe rolle taufe teil """.split()) def capitalized(word): """The word with only its first letter uppercased, or None (ß has no capital form).""" first = word[0].upper() return first + word[1:] if len(first) == 1 else None def model_words(path): """The words of an HXTM v3 model, read as build-typing-model.py writes them.""" data = open(path, "rb").read() if data[:4] != b"HXTM": raise ValueError(path + ": not an HXTM file") version, words, pairs, blob, alphabet = struct.unpack_from(" CAP: # A two-letter "word" is an abbreviation ("gl", "rn"), never a noun. if len(w) == 2: reason = "short" elif w in ALSO_LOWERCASE: reason = "list" elif verb_form(w, low, cap): reason = "verb" elif stem_form(w, low, cap): reason = "stem" else: reason = None if reason: p = CAP capped.append((w, total, reason)) rows.append((w, p)) rows.sort() with open(out_path + ".part", "w", encoding="utf-8", newline="\n") as f: for w, p in rows: f.write(f"{w}\t{p}\n") os.replace(out_path + ".part", out_path) high = sum(1 for _, p in rows if p >= CAP + 1) capped.sort(key=lambda row: -row[1]) counts_by_reason = { reason: sum(1 for _, _, r in capped if r == reason) for reason in ("verb", "stem", "list", "short") } print(f"{out_path}: {len(rows):,} of {len(words):,} words, {high:,} at 95% or more, " f"{len(rows) - high:,} at 50-94%, {len(counts):,} tokens summed") print(f"{len(capped):,} words moved from 95% or more to {CAP}% " f"({counts_by_reason['verb']} verb forms, {counts_by_reason['stem']} stems, " f"{counts_by_reason['list']} listed, {counts_by_reason['short']} short):") for w, total, reason in capped[:250]: print(f" {w} ({total:,}, {reason})") if __name__ == "__main__": if len(sys.argv) != 3: sys.exit(__doc__) try: build(*sys.argv[1:]) except BaseException: print(f"leaving {WORK} for a resume", file=sys.stderr) raise