# Code examples from: Build Your Own LLM, Lesson 3: A Language Model Made of Counting
# https://www.atulvij.com/llm-course-3-bigram-model
# Review and adapt before running; examples use sample values.

# ---- Example 1 (python) ----
text = open("input.txt", encoding="utf-8").read()
chars = sorted(set(text))
print(len(text), "characters,", len(chars), "unique")

# ---- Example 2 (python) ----
from collections import defaultdict, Counter

counts = defaultdict(Counter)
for a, b in zip(text, text[1:]):
    counts[a][b] += 1

print(counts["q"].most_common(3))   # in English, almost always 'u'
print(counts[" "].most_common(5))   # common first letters of words

# ---- Example 3 (python) ----
import random

def next_char(c):
    options = counts[c]
    return random.choices(list(options), weights=list(options.values()))[0]

c = "T"
out = [c]
for _ in range(300):
    c = next_char(c)
    out.append(c)
print("".join(out))

# ---- Example 4 (python) ----
import math

totals = {a: sum(c.values()) for a, c in counts.items()}
nll, n = 0.0, 0
for a, b in zip(text, text[1:]):
    p = counts[a][b] / totals[a]
    nll += -math.log(p)
    n += 1
print("average loss:", round(nll / n, 3))
print("random-guess loss:", round(math.log(len(chars)), 3))

# ---- Example 5 (python) ----
def prob(a, b):
    return (counts[a][b] + 1) / (totals.get(a, 0) + len(chars))