diff --git a/apps/website/i18n/neural-networks.ru.json b/apps/website/i18n/neural-networks.ru.json
new file mode 100644
index 0000000000..dc0bec6914
--- /dev/null
+++ b/apps/website/i18n/neural-networks.ru.json
@@ -0,0 +1,242 @@
+{
+ "$spec": "specs/course/neural-networks-ru.t27",
+ "$note": "Russian words for specs/course/neural-networks.t27. Keys are MODULE_IDS, LESSON_IDS and, under ui, the SAY_ names. Numbers and {placeholders} must match the English field; course-from-spec.mjs checks both.",
+ "locale": "ru",
+ "ui": {
+ "TITLE": "Нейросети на t27: от одного нейрона до блока трансформера",
+ "DESCRIPTION": "Курс 10, 27 уроков, в каждом живой виджет и спека t27: функции активации, softmax и эмбеддинги, пулинг и пакетная нормализация, функции потерь, SGD, RMSProp и Adam, обратное распространение и XOR, рекуррентные ячейки, внимание, блок трансформера и PPO. Каждое число вычисляет скомпилированная спека.",
+ "KICKER": "Курс",
+ "LEAD": "Агенты этого проекта думают нейросетями, а здесь их части записаны спеками t27 с тестами, которые проверяет компилятор. Каждый виджет исполняет тела fn спеки из specs/ml, скомпилированные в WebAssembly: двигаете ползунок — отвечает спека.",
+ "SHAPE": "9 модулей по 3 урока, 27 клеток. Заполненная клетка — урок, отмеченный пройденным.",
+ "START": "Начать урок 1",
+ "CONTINUE": "Продолжить",
+ "MODULE": "Модуль {0}",
+ "LESSON": "Урок {0} из {1}",
+ "GOAL": "Вы узнаете",
+ "TRY": "Попробовать",
+ "ALSO": "Также откройте",
+ "ALSO_HINT": "Каждая кнопка меняет виджет сверху.",
+ "BACK_TO_MAIN": "Вернуться к виджету урока",
+ "SPEC": "Открыть спеку урока в плеере",
+ "OPEN_PAGE": "Открыть виджет на отдельной странице",
+ "WIDGET_LANG": "Виджеты хранят свои английские слова: у каждого своя спека t27.",
+ "PREV": "Назад",
+ "NEXT": "Далее",
+ "ALL": "Все уроки",
+ "MARK": "Отметить пройденным",
+ "MARKED": "Пройдено",
+ "PROGRESS": "{0} из {1} пройдено",
+ "PRIVATE": "Прогресс хранится только в этом браузере; ничего не отправляется.",
+ "SOURCE": "Этот курс сам является спекой t27: прочитайте её",
+ "COHORT": "Ищете платное обучение по FPGA? Оно переехало на отдельную страницу.",
+ "NOT_FOUND": "У урока нет такого адреса.",
+ "OPEN_LESSON": "Открыть интерактивный урок",
+ "OPEN_COURSE": "Открыть интерактивный курс",
+ "SEO_TITLE": "{0}: курс о нейросетях, урок {1} из {2}",
+ "SHARE": "Ссылка для шаринга, с карточкой предпросмотра",
+ "NEXT_COURSE": "Следующий курс",
+ "PREV_COURSE": "Предыдущий курс"
+ },
+ "modules": {
+ "one-neuron": {
+ "TITLE": "Один нейрон",
+ "LINE": "Нейрон складывает входы и изгибает сумму: ReLU, сигмоида и tanh, и где каждая перестаёт пропускать градиент назад."
+ },
+ "new-bends": {
+ "TITLE": "Новые изгибы",
+ "LINE": "Гладкие кривые нынешних моделей: GELU с провалом, Swish с ручкой и ELU с полом вместо обрыва."
+ },
+ "a-layer": {
+ "TITLE": "Слой",
+ "LINE": "Что окружает нейроны: softmax превращает оценки в выбор, эмбеддинг превращает слово в числа, dropout выключает нейроны."
+ },
+ "pictures": {
+ "TITLE": "Картинки",
+ "LINE": "Чтение картинки по участкам: окно, усреднённое по участку, max pooling и пакетная нормализация признаков разного масштаба."
+ },
+ "the-error": {
+ "TITLE": "Ошибка",
+ "LINE": "Насколько неверна догадка, одним числом: среднеквадратичная ошибка, перекрёстная энтропия для вероятностей и функция Хьюбера для выбросов."
+ },
+ "downhill": {
+ "TITLE": "Под гору",
+ "LINE": "Спуск по функции потерь: градиентный спуск с моментом, RMSProp без оглядки на величину градиента и Adam, который соединяет оба."
+ },
+ "learning": {
+ "TITLE": "Обучение",
+ "LINE": "Откуда берутся градиенты и чего они добиваются: обратное распространение через слой, сеть, которая учит XOR, и LAMB для больших пакетов."
+ },
+ "memory-and-attention": {
+ "TITLE": "Память и внимание",
+ "LINE": "Сети, которые читают последовательность: рекуррентная ячейка и её затухающее эхо, вентили LSTM и внимание ко всем позициям сразу."
+ },
+ "transformer-and-reward": {
+ "TITLE": "Трансформер и награда",
+ "LINE": "Части трансформера и агента: позиции как часы, один блок энкодера и PPO, который учится на награде без рывков."
+ }
+ },
+ "lessons": {
+ "relu": {
+ "TITLE": "ReLU: оставить положительную часть",
+ "GOAL": "Что ReLU делает с положительным и с отрицательным входом и почему наклон 0 останавливает обучение нейрона.",
+ "TEXT": "Нейрон складывает взвешенные входы и пропускает сумму через функцию активации. ReLU из них самая простая, max(0, x): положительная сумма проходит без изменений, отрицательная становится 0. Её наклон равен 1 или 0, а наклон 0 означает нейрон, который перестал учиться: градиент через него назад не течёт. Утечка это исправляет: с наклоном 0.1 вход x = -2 даёт -0.2 и пропускает назад градиент 0.1. Спека relu_activation.t27 вычисляет обе кривые, которые рисует виджет.",
+ "TASK": "Поставьте x на -2 и прочитайте наклон с утечкой и без неё; затем найдите наклон при x = 1.5."
+ },
+ "the-sigmoid": {
+ "TITLE": "Сигмоида: число в вероятность",
+ "GOAL": "Как сигмоида переводит любой вход в вероятность и почему её наклон мешает глубоким сетям.",
+ "TEXT": "Сигмоида 1 / (1 + e^-x) сжимает всю числовую прямую в отрезок от 0 до 1, поэтому её читают как вероятность: 2 превращается в 0.8808, а -4 в 0.018. Её наклон равен s (1 - s) и считается по самому выходу s; он наибольший в 0, где равен 0.25, а при x = 8 равен 0.000335. Градиент, который идёт назад через стопку сигмоид, на каждой умножается не больше чем на 0.25, поэтому глубокие сети на сигмоидах учились медленно. Спека sigmoid_activation.t27 вычисляет кривую и наклон.",
+ "TASK": "Найдите вход, при котором вероятность равна 0.5; затем перетащите к 6 и посмотрите, как мало осталось наклона."
+ },
+ "tanh-and-saturation": {
+ "TITLE": "tanh и насыщение",
+ "GOAL": "Почему tanh центрирована на нуле и где её наклон падает так низко, что нейрон перестаёт учиться.",
+ "TEXT": "tanh это сигмоида, растянутая на -1..1 и центрированная на 0, поэтому слой нейронов tanh передаёт дальше значения со средним около нуля, а её наклон в центре достигает 1. Она тоже насыщается: при 2.5 выход 0.9866, а наклон всего 0.0266. Красная полоса отмечает все входы, где наклон меньше 0.1, дальше примерно 1.8 в обе стороны, и нейрон, чьи входы живут там, почти не учится. Спека tanh_activation.t27 вычисляет кривую, наклон и свой пакет, tanh от 0, 1 и 2: 0, 0.762 и 0.964.",
+ "TASK": "Тяните x, пока не начнётся полоса, и прочитайте наклон там; затем сравните наклон в 0 и при 2.5."
+ },
+ "gelu": {
+ "TITLE": "GELU: провал ниже нуля",
+ "GOAL": "Что вычисляет GELU, почему она проваливается ниже нуля и насколько близко подходит её приближение через tanh.",
+ "TEXT": "GELU взвешивает x вероятностью того, что стандартная нормальная величина меньше него: GELU(x) = x Phi(x). Большой положительный вход проходит, большой отрицательный исчезает, а между ними кривая проваливается примерно до -0.17 около x = -0.75 и только потом растёт. BERT и GPT используют её в своих слоях прямого распространения. Точной форме нужна erf, и спека вычисляет её приближением Абрамовица-Стиган с точностью 1.5e-7, а также вычисляет приближение через tanh: при -0.75 они дают -0.16997 и -0.17004. Спека gelu_activation.t27 вычисляет обе кривые и их наклоны.",
+ "TASK": "Найдите нижнюю точку кривой по её наклону; затем сравните точную форму и форму через tanh при -3."
+ },
+ "swish-and-silu": {
+ "TITLE": "Swish и SiLU",
+ "GOAL": "Как x, умноженный на sigmoid(beta x), открывает сам себя и как один параметр переводит его от гладкой кривой к ReLU.",
+ "TEXT": "Swish это x, умноженный на sigmoid(beta x): нейрон, который сам себе вентиль. При beta 1 он называется SiLU, и его нижняя точка -0.2785 около x = -1.28. Увеличьте beta, и вентиль становится резче: при beta 8 кривая это ReLU со скруглёнными углами, при beta 0.5 пологий склон через ноль. Один параметр переводит нейрон от гладкой кривой к переключателю, и сеть может его выучить. Спека silu_swish_activation.t27 вычисляет кривую и её наклон при любом beta, а также свой пакет при -2, 0 и 2: -0.238, 0 и 1.762.",
+ "TASK": "Держите x на -1.28 и поднимайте beta от 1 до 8; затем найдите beta, при котором провал на этом масштабе исчезает."
+ },
+ "elu": {
+ "TITLE": "ELU: пол, а не обрыв",
+ "GOAL": "Как ELU сохраняет мягкую отрицательную сторону с полом на -alpha и почему её наклон никогда не умирает до конца.",
+ "TEXT": "ELU сохраняет положительную сторону ReLU и заменяет обрыв на отрицательной стороне кривой alpha (e^x - 1), которая выполаживается к -alpha. При alpha 1 вход x = -3 даёт -0.9502, уже близко к полу -1, а наклон там 0.0498: мал, но не 0, так что нейрон ещё может сдвинуться. Отрицательные выходы не выбрасываются, поэтому среднее выходов слоя ближе к 0. Спека elu_activation.t27 вычисляет кривую, наклон и пол, а также свой пакет при -1, 0 и 2: -0.632, 0 и 2.",
+ "TASK": "Поставьте alpha на 0.5 и прочитайте пол; затем найдите, как далеко влево должен уйти x, чтобы наклон стал меньше 0.01."
+ },
+ "softmax-and-temperature": {
+ "TITLE": "Softmax и температура",
+ "GOAL": "Как softmax превращает оценки в вероятности с суммой один и что с ними делает температура.",
+ "TEXT": "Классификатор заканчивается одной оценкой на класс, и softmax превращает оценки в вероятности: exp каждой оценки, делённая на сумму всех. Оценки 1, 2 и 3 становятся 0.090, 0.245 и 0.665. Если сначала разделить оценки на температуру, меняется уверенность: при 0.5 верхняя оценка забирает 0.867, при 5 все три равны 0.27, 0.33 и 0.40. Языковая модель выбирает следующий токен именно отсюда, и настройка температуры чат-модели это и есть T. Спека softmax.t27 сначала вычитает наибольшую оценку, поэтому exp никогда не переполняется.",
+ "TASK": "Охладите температуру до 0.1 и прочитайте три вероятности; затем сделайте ничью и нагрейте её."
+ },
+ "embeddings": {
+ "TITLE": "Эмбеддинги",
+ "GOAL": "Как токен превращается в строку чисел и почему обучение меняет только прочитанные строки.",
+ "TEXT": "Сеть не умеет умножать слово, поэтому каждый токен это индекс в таблице с одной строкой чисел на токен, а поиск токена копирует его строку. В спеке в таблице 4 токена шириной 2, и строка v хранит (10 v, 10 v + 1), так что токен 2 читает (20, 21). Обучение отправляет градиент назад только в прочитанные строки, а токен, прочитанный дважды, собирает оба: строки градиента (1, 2) и (3, 4) на токене 2 складываются в (4, 6). Спека embedding_layer.t27 вычисляет поиск и градиент.",
+ "TASK": "Прочитайте токен 2 дважды, а затем токены 1 и 3; каждый раз сосчитайте строки, где градиент не 0."
+ },
+ "dropout": {
+ "TITLE": "Dropout: выключенные нейроны",
+ "GOAL": "Почему выключение нейронов во время обучения помогает и почему выживших нужно масштабировать.",
+ "TEXT": "Во время обучения dropout случайно выключает долю p нейронов, чтобы остальные не опирались ни на один из них. Выжившие нейроны тогда нужно масштабировать, иначе следующий слой видел бы сигнал другого размера при обучении, чем при выводе. Обратный dropout умножает их на 1 / (1 - p) при обучении: с p = 0.5 вход 1, 2, 3, 4 становится 2, 0, 6, 0, а вывод это простой проход. Классический dropout оставляет 1, 0, 3, 0 и умножает на 0.5 уже при выводе. Спека dropout_layer.t27 использует фиксированную маску, оставить, выбросить, оставить, выбросить, поэтому числа каждый раз одинаковые.",
+ "TASK": "Сравните выходы четырёх режимов; затем прочитайте градиент выброшенного нейрона при обучении."
+ },
+ "patches-of-a-picture": {
+ "TITLE": "Участки картинки",
+ "GOAL": "Почему сети читают картинку по участкам и что даёт окно 2 x 2, усреднённое по каждому участку.",
+ "TEXT": "Картинка это сетка чисел, и сети читают её небольшими участками, чтобы один и тот же детектор работал в любом её месте. Здесь окно 2 x 2 шагает по 2 по картинке 4 x 4, где в клетке i лежит i, и каждый выход это среднее своего участка: 2.5, 4.5, 10.5 и 12.5. При обратном проходе градиент каждого выхода делится поровну на его 4 клетки. Это усредняющий пулинг: свёртка взвешивала бы 4 клетки выученным ядром, и её спека, t27#10612, ещё не слита. Спека avgpool2d_layer.t27 вычисляет оба прохода.",
+ "TASK": "Нажмите на каждый участок и проверьте его среднее на глаз; затем включите градиент и прочитайте, что получает каждая клетка окна 4."
+ },
+ "max-pooling": {
+ "TITLE": "Max pooling: максимум окна",
+ "GOAL": "Что оставляет max pooling, откуда, как он помнит, это взято и куда идёт градиент.",
+ "TEXT": "Max pooling оставляет наибольшее значение каждого окна и отбрасывает остальные, поэтому сети становится менее важно, где именно лежит признак. На сетке спеки 4 x 4 окна 2 x 2 с шагом 2 оставляют 4, 8, 9 и -2, взятые из клеток 5, 6, 13 и 11. Перекрывающиеся окна 3 x 3 с шагом 1 оставляют 8, 8, 9 и 9. При обратном проходе весь градиент окна идёт в его максимум: градиенты 1, 2, 3 и 4 попадают в клетки 5, 6, 13 и 11, а все остальные клетки получают 0. Спека maxpool2d_layer.t27 вычисляет выходы, их позиции и градиент.",
+ "TASK": "Переключитесь на окна 3 x 3 и найдите клетку, общую для двух выходов; затем включите градиент для окон 2 x 2."
+ },
+ "batch-normalisation": {
+ "TITLE": "Пакетная нормализация",
+ "GOAL": "Как пакетная нормализация приводит признаки разного масштаба к одному и чем она пользуется при выводе.",
+ "TEXT": "Признаки разного масштаба дёргают сеть: в пакете спеки один признак в десять раз больше другого. Пакетная нормализация вычитает среднее каждого признака по пакету и делит на его разброс, поэтому признаки 1, 2, 3, 4 и 10, 20, 30, 40 оба выходят как -1.34, -0.45, 0.45 и 1.34, а -1, -1, 1, 1 выходит как -1 и 1. Затем выученные gamma и beta масштабируют и сдвигают его обратно, где это полезно. При выводе слой вместо пакета использует скользящие средние: при среднем 0, дисперсии 1, gamma 1 и beta 0.25 входы 1, 2 и 3 дают 1.25, 2.25 и 3.25. Спека batchnorm_layer.t27 вычисляет оба режима.",
+ "TASK": "Сравните два нормализованных столбца; затем поставьте скользящее среднее 2 и дисперсию 4 и прочитайте выходы вывода."
+ },
+ "mean-squared-error": {
+ "TITLE": "Среднеквадратичная ошибка",
+ "GOAL": "Как MSE превращает каждый промах в одно число, почему она гоняется за большими ошибками и куда смотрит её градиент.",
+ "TEXT": "Функция потерь превращает все промахи модели в одно число, которое нужно уменьшать. Среднеквадратичная ошибка усредняет квадраты промахов: предсказания 1.5, 2 и 2 против целей 1, 2 и 3 стоят (0.25 + 0 + 1) / 3 = 0.4167. Из-за квадрата промах на 2 стоит в 4 раза больше промаха на 1, поэтому MSE гоняется за большими ошибками. Её градиент для каждого предсказания равен 2 (p - t) / n, здесь 0.333, 0 и -0.667: каждое предсказание толкается обратно к своей цели. Как функция одного предсказания потеря это парабола, чаша с одним дном. Спека mse_loss.t27 вычисляет потерю и градиент.",
+ "TASK": "Передвиньте предсказание 2 на его цель и прочитайте потерю; затем ведите по чаше и найдите её дно."
+ },
+ "cross-entropy": {
+ "TITLE": "Перекрёстная энтропия",
+ "GOAL": "Почему перекрёстная энтропия так дорого берёт за уверенный неверный ответ и почему именно это учит классификатор.",
+ "TEXT": "Классификатор выдаёт вероятность, а перекрёстная энтропия берёт -log вероятности, которую он дал верному ответу. Предсказания 0.8, 0.3 и 0.9 против ответов 1, 0 и 1 стоят в среднем 0.2284. Быть уверенным и правым почти бесплатно: 0.99, 0.01 и 0.99 стоят 0.0101. Быть уверенным и неправым дорого: 0.01, 0.99 и 0.01 стоят 4.6052, а к 0 стоимость растёт без предела, поэтому спека обрезает p до [1e-7, 1 - 1e-7]. Её градиент наибольший как раз там, где модель уверена и неправа. Спека binary_crossentropy_loss.t27 вычисляет потерю и градиент, из вероятностей или из логитов.",
+ "TASK": "Тяните предсказание 0 к 0.01 и смотрите на кривую; затем прочитайте столбики градиента при 0.5, 0.5 и 0.5."
+ },
+ "the-huber-loss": {
+ "TITLE": "Функция Хьюбера",
+ "GOAL": "Как функция Хьюбера обходится с малой ошибкой как с квадратичной, а с большой линейно, чтобы выбросы тянули слабее.",
+ "TEXT": "Квадратичная ошибка позволяет одному выбросу всё решать: ошибка 3 стоит 4.5 как половина своего квадрата, столько же, сколько 36 ошибок по 0.5. Функция Хьюбера равна половине квадрата, пока |e| не больше delta, а дальше растёт только линейно, delta |e| - delta^2 / 2, так что при delta 1 та же ошибка стоит 2.5. Два куска сходятся с одинаковым значением и наклоном, и градиент никогда не превышает delta: далёкая точка тянет модель с ограниченной силой. Для пары ошибок спеки 0.5 и 3 среднее равно 1.3125 против 2.3125 для половины квадрата. Спека huber_loss.t27 вычисляет обе.",
+ "TASK": "Поставьте delta на 0.5 и тяните ошибку к 3; затем найдите, где две кривые расходятся."
+ },
+ "gradient-descent": {
+ "TITLE": "Градиентный спуск",
+ "GOAL": "Как градиентный спуск идёт вниз по функции потерь, что добавляет момент и почему слишком большой шаг никогда не успокаивается.",
+ "TEXT": "Обучение это спуск под гору по функции потерь. Градиентный спуск каждый шаг сдвигает вес на -lr, умноженное на его градиент; в спеке потеря равна (x - 3)^2 от x = 0, и lr 0.1 доходит до 2.99996 за 50 шагов. Момент добавляет долю прошлого сдвига, что ускоряет длинный склон и перелетает острое дно: при 0.9 вес пролетает 3 примерно до 5.13 и, раскачиваясь, приходит к 3.09 после 50 шагов. Нестеров берёт градиент на шаг вперёд и качается меньше, примерно до 4.25. Слишком большой шаг никогда не успокаивается: при lr 1 вес навсегда скачет между 0 и 6. Спека sgd.t27 вычисляет каждый шаг.",
+ "TASK": "Найдите наибольший шаг обучения, который ещё успокаивается без момента; затем добавьте момент 0.9 и сосчитайте качания."
+ },
+ "rmsprop": {
+ "TITLE": "RMSProp: шаг без масштаба",
+ "GOAL": "Как RMSProp делит каждый шаг на скользящий корень среднего квадрата, чтобы шаг больше не зависел от величины градиента.",
+ "TEXT": "Правильный размер шага зависит от величины градиента, а разные веса видят градиенты, отличающиеся в миллион раз. RMSProp хранит скользящее среднее квадрата градиента, кэш, и делит каждый шаг на его корень, поэтому у шага нет единиц: при постоянном градиенте 0.001, 0.5 или 1000 вес на шаге 1 сдвигается ровно на те же 0.0316 и идёт тем же путём, доходя до -0.0914 после 100 шагов при lr 0.01. Затухание задаёт, как медленно наполняется кэш: при 0.99 те же 100 шагов заканчиваются на -1.0261. Спека rmsprop.t27 вычисляет вес и кэш.",
+ "TASK": "Переключайтесь между тремя градиентами и убедитесь, что линии не двигаются; затем поднимите затухание до 0.99."
+ },
+ "adam": {
+ "TITLE": "Adam: момент и масштаб",
+ "GOAL": "Как Adam соединяет момент со скользящим масштабом и почему поправка смещения делает шаг 1 фиксированного размера.",
+ "TEXT": "Adam соединяет обе идеи: m, скользящее среднее градиента, как в моменте, и v, скользящее среднее его квадрата, как в RMSProp; шаг равен lr m / sqrt(v). Оба начинают с 0 и поначалу слишком малы, поэтому Adam делит их на 1 - beta^t, и на шаге 1 это делает сдвиг ровно lr, умноженным на знак градиента: градиенты 2, -40 и 0.5 сдвигают свои веса на одинаковые 0.001. На чаше спеки (x - 3)^2 при lr 0.1 Adam поднимается от 0, перелетает примерно до 3.18 и успокаивается на 3.00005 к шагу 200. Спека adam.t27 вычисляет каждый шаг.",
+ "TASK": "Дайте трём градиентам очень разные размеры и прочитайте веса после шага 1; затем найдите, где путь достигает вершины."
+ },
+ "backpropagation": {
+ "TITLE": "Обратное распространение",
+ "GOAL": "Как градиент каждого веса и каждого входа слоя следует из градиента его выходов.",
+ "TEXT": "Чтобы учиться, каждому весу нужно знать, как меняется потеря, когда он сдвигается. Обратное распространение отвечает по одному слою за раз: при данном g, насколько потеря хочет сдвинуть каждый выход, вес от входа i к выходу o получает g_o x_i, а вход i получает сумму по o от W[o][i] g_o и передаёт её слою ниже. При входах (1, 2, 3) и g = (1, 0.5) градиенты весов равны 1, 2, 3 и 0.5, 1, 1.5, а градиенты входов 0.5, 2 и 3.5. Слой линейный, поэтому сдвиг x0 на 1 сдвигает потерю g . y с 15.25 до 15.75, на его градиент 0.5. Спека dense_layer.t27 вычисляет все три.",
+ "TASK": "Поставьте g = (1, 0) и посмотрите, какие рёбра гаснут; затем поставьте g = (-2, 1) и проверьте градиенты входов вручную."
+ },
+ "learning-xor": {
+ "TITLE": "Учим XOR",
+ "GOAL": "Почему один нейрон не может выучить XOR и как это делает скрытый слой, обученный обратным распространением.",
+ "TEXT": "XOR равен 1, когда ровно один из двух входов равен 1. Никакая прямая не отделяет случаи, дающие 1, от дающих 0, поэтому один нейрон выучить его не может. Скрытый слой изгибает пространство: спека обучает сеть 2-8-1 из ReLU обратным распространением, lr 0.05, из фиксированных начальных весов. Её потеря падает с 0.8896 в эпохе 1 до 1.8e-13 после 2000 эпох, а четыре ответа выходят в пределах 4e-7 от 0, 1, 1 и 0. Виджет проводит это обучение в вашем браузере, заново для каждой точки кривой. Спека mlp.t27 хранит сеть и тест, что она учится.",
+ "TASK": "Отведите эпохи назад к 100 и прочитайте четыре ответа; затем найдите эпоху, где кривая перестаёт падать."
+ },
+ "lamb-and-large-batches": {
+ "TITLE": "LAMB и большие пакеты",
+ "GOAL": "Как LAMB масштабирует шаг каждого слоя его собственной нормой весов, чтобы один шаг обучения подходил слоям любого размера.",
+ "TEXT": "Обучение на очень больших пакетах требует большого шага обучения, а один шаг может быть слишком большим для малого слоя и слишком малым для большого. LAMB берёт обновление в духе Adam для каждого слоя и перемасштабирует его коэффициентом доверия, нормой весов слоя, ограниченной 10, делённой на норму обновления, поэтому каждый слой сдвигается на одну и ту же долю самого себя. В спеке веса (1, 2), умноженные на масштаб, делают шаг с нормой, в 0.001 раз меньшей их собственной: 0.00447 при масштабе 2, а когда норма весов переходит 10, около масштаба 4.47, шаг остаётся на 0.01. Спека lamb.t27 вычисляет обновление для одного слоя и для двух сразу.",
+ "TASK": "Прочитайте шаг при масштабах 1 и 2 и сравните; затем перетащите за изгиб и объясните, почему линия становится плоской."
+ },
+ "a-recurrent-cell": {
+ "TITLE": "Рекуррентная ячейка",
+ "GOAL": "Как рекуррентная ячейка переносит состояние от шага к шагу и почему эта память затухает.",
+ "TEXT": "Предложение приходит по одному слову, и рекуррентная ячейка переносит состояние h от шага к шагу: h' = tanh(W_x x + W_h h + b). В крошечной ячейке спеки W_x равен 1, а W_h 0.5, поэтому один импульс даёт 0.7616, 0.3634 и 0.1797: память о нём примерно вдвое слабеет каждый шаг. Второй импульс на шаге 3 поднимает h до 0.7968, чуть выше, чем начинался одиночный импульс, потому что старая память ещё там. После 8 шагов остаётся лишь 0.0466. Это затухание и есть причина, по которой простые ячейки забывают длинный контекст. Спека rnn_cell.t27 вычисляет каждый шаг.",
+ "TASK": "Загрузите один импульс и прочитайте, за сколько шагов он падает ниже 0.1; затем сделайте вход чередующимся."
+ },
+ "lstm-gates": {
+ "TITLE": "Вентили LSTM",
+ "GOAL": "Как вентили забывания, входа и выхода LSTM решают, что ячейка хранит, записывает и показывает.",
+ "TEXT": "Ячейка LSTM хранит отдельную память c и три вентиля, каждый это сигмоида между 0 и 1: забывание масштабирует старую память, вход впускает новое содержимое g, а выход решает, сколько памяти показать. c' = f c + i g и h' = o tanh(c'). В фикстуре спеки каждый вес равен 0, поэтому каждый вентиль это сигмоида своего смещения: при вентиле забывания 0.9975 и вентиле входа 0.0025 память 1 выходит как 0.9975, а выход равен 0.7587. Так как c проходит лишь с умножением на f, её градиент тоже переживает много шагов. Спека lstm_single.t27 вычисляет ячейку.",
+ "TASK": "Загрузите Remember, а затем Forget и сравните новую память; затем запишите с открытым вентилем входа."
+ },
+ "attention": {
+ "TITLE": "Внимание",
+ "GOAL": "Как внимание сравнивает каждый запрос с каждым ключом, что даёт softmax по каждой строке и что убирает причинная маска.",
+ "TEXT": "Внимание позволяет каждой позиции решить, какие другие позиции читать. Каждый запрос сравнивается с каждым ключом, q . k / sqrt(d), и softmax превращает каждую строку оценок в веса с суммой 1; выход это значения, усреднённые этими весами. Две головы делают это на двух половинах вектора. С причинной маской позиция не может читать будущее: в голове 0 позиция 0 читает только себя, позиция 1 делит 0.452 и 0.548, а позиция 2 распределяет 0.230, 0.321 и 0.449. Без маски каждая строка распределяется по всем трём ключам, от 0.21 до 0.47. Спека attention_mechanism.t27 вычисляет каждый вес на своей фикстуре.",
+ "TASK": "Выключите и включите маску и прочитайте строку 0; затем сравните две головы в строке 2."
+ },
+ "positional-encoding": {
+ "TITLE": "Позиционное кодирование",
+ "GOAL": "Как синусы и косинусы убывающих частот дают каждой позиции свой вектор и почему сдвиг это поворот.",
+ "TEXT": "Внимание видит свой вход как множество, поэтому порядок слов нужно к ним добавить. Синусоидальное кодирование даёт каждой позиции d чисел: sin и cos позиции, умноженной на частоту, которая от пары к паре падает в 10000^(2/d) раз. При d = 8 четыре пары поворачиваются на 1, 0.1, 0.01 и 0.001 радиана за позицию, как стрелки четырёх часов: позиция 5 читает -0.959, 0.284 в быстрой паре и 0.005, 1.000 в медленной. Длина каждой пары равна 1, и сдвиг на k позиций поворачивает каждую пару на свой постоянный угол. Спека positional_encoding.t27 вычисляет каждое значение.",
+ "TASK": "Шагните с позиции 0 на 1 и посмотрите, какие часы двигаются; затем перейдите на 37 и найдите пару, которая всё ещё смотрит вверх."
+ },
+ "an-encoder-block": {
+ "TITLE": "Блок энкодера",
+ "GOAL": "Что один блок энкодера трансформера делает со своим входом и что меняет перенос нормализаций вперёд.",
+ "TEXT": "Слой трансформера это два подслоя, каждый обёрнут остаточным сложением и нормализацией слоя: самовнимание смешивает позиции, затем сеть прямого распространения ширины 8 работает с каждой позицией отдельно. В порядке post-norm из исходной статьи блок кончается нормализацией, поэтому у каждой строки выхода среднее 0 и дисперсия 1.000. Pre-norm ставит нормализации перед каждым подслоем и кончается остаточным сложением, поэтому его строки выходят с дисперсиями 2.73, 3.33 и 0.96; глубокие стопки так учатся легче, с одной нормализацией в самом верху. Спека encoder_block.t27 прогоняет один блок на 3 позициях, d_model 4 и 2 головах.",
+ "TASK": "Переключайтесь между post-norm и pre-norm и читайте дисперсии строк; затем найдите позицию, которую голова 1 читает больше всего."
+ },
+ "ppo-and-the-reward": {
+ "TITLE": "PPO и награда",
+ "GOAL": "Как PPO превращает награду в обновление политики и как его обрезка не даёт одному обновлению зайти слишком далеко.",
+ "TEXT": "Агент учится на награде, делая хорошие действия вероятнее. PPO измеряет изменение политики отношением r новой вероятности действия к старой и умножает его на преимущество A, насколько действие превзошло ожидание. Затем он обрезает: когда r ушло больше чем на eps в ту сторону, за которую награждает A, цель становится плоской. При r = 1.5, A = 1 и eps 0.2 сырое r A равно 1.5, но PPO засчитывает 1.2, так что один пакет не может далеко сдвинуть политику. На пакете спеки, отношения 1, 1.1 и 0.9 с преимуществами 2, 1.5 и 3, потеря политики равна -2.1167. Спека ppo_clip_loss.t27 её вычисляет.",
+ "TASK": "Сделайте преимущество отрицательным и найдите, где кривая теперь становится плоской; затем расширьте eps до 0.5."
+ }
+ }
+}
diff --git a/apps/website/public/learn/a-recurrent-cell/card.png b/apps/website/public/learn/a-recurrent-cell/card.png
new file mode 100644
index 0000000000..54d84d1caa
Binary files /dev/null and b/apps/website/public/learn/a-recurrent-cell/card.png differ
diff --git a/apps/website/public/learn/a-recurrent-cell/index.html b/apps/website/public/learn/a-recurrent-cell/index.html
new file mode 100644
index 0000000000..0eca5288aa
--- /dev/null
+++ b/apps/website/public/learn/a-recurrent-cell/index.html
@@ -0,0 +1,265 @@
+
+
+
+
+
+
+A recurrent cell: neural networks course, lesson 22 of 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiРусский
+
+
+
A recurrent cell
+
You will learn
How a recurrent cell carries a state from step to step, and why that memory fades.
+
A sentence comes one word at a time, and a recurrent cell carries a state h from one step to the next: h' = tanh(W_x x + W_h h + b). In the spec's tiny cell W_x is 1 and W_h 0.5, so a single pulse gives 0.7616, 0.3634 and 0.1797: the memory of it roughly halves each step. A second pulse at step 3 lifts h to 0.7968, a little above where the single pulse began, because the old memory is still there. After 8 steps only 0.0466 is left. That fading is why plain cells forget long contexts. The spec rnn_cell.t27 computes each step.
+
Try it
Load one pulse and read how many steps it takes to fall under 0.1; then make the input alternate.
Tap the inputs of eight steps: the cell feeds its own last output back, h' = tanh(x + 0.5 h), so every pulse echoes on and fades by about half a step. Two pulses add up.
+
specs/ml/recurrent/rnn_cell.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Basic RNN recurrence equation | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written against undeclared `matmul`, `add`, `zeros`, `len` and `append`,
+// it applied tanh twice (h = tanh(tanh(z))), and the tests asserted
+// `result != undefined`. It is now the Elman cell,
+// h' = tanh(W_x x + W_h h + b), with W_x hidden x input and W_h hidden x hidden
+// (row-major), stepping over buffers the caller owns.
+
+module RnnCell;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_INPUT_SIZE : u32 = 128;
+ const DEFAULT_HIDDEN_SIZE : u32 = 128;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const RNNCellConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ };
+
+ pub const RNNState = struct {
+ hidden : []f32,
+ cell : []f32,
+ };
+
+ pub const RNNParams = struct {
+ w_x : []f32,
+ w_h : []f32,
+ b : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // new_hidden[i] = tanh(sum_j W_x[i][j] x[j] + sum_k W_h[i][k] h[k] + b[i])
+ fn forward_step(input: []const f32, hidden: []const f32, params: RNNParams, config: RNNCellConfig, new_hidden: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.hidden_size) {
+ var z : f32 = params.b[i];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + params.w_x[i * config.input_size + j] * input[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + params.w_h[i * config.hidden_size + k] * hidden[k];
+ k = k + 1;
+ }
+ new_hidden[i] = tanh(z);
+ i = i + 1;
+ }
+ }
+
+ // inputs: steps x input_size; outputs: steps x hidden_size, row t = h after step t.
+ // The state starts at initial_state and each step reads the previous row.
+ fn forward_sequence(inputs: []const f32, steps: usize, initial_state: []const f32, params: RNNParams, config: RNNCellConfig, outputs: []f32) -> void {
+ const n_in : usize = config.input_size;
+ const n_h : usize = config.hidden_size;
+ var t : usize = 0;
+ while (t < steps) {
+ if (t == 0) {
+ forward_step(inputs[0..n_in], initial_state, params, config, outputs[0..n_h]);
+ } else {
+ forward_step(inputs[t * n_in .. (t + 1) * n_in], outputs[(t - 1) * n_h .. t * n_h], params, config, outputs[t * n_h .. (t + 1) * n_h]);
+ }
+ t = t + 1;
+ }
+ }
+
+ // The first hidden_size entries of `buffer`, zeroed
+ fn init_state(buffer: []f32, hidden_size: u32) -> []f32 {
+ var i : usize = 0;
+ while (i < hidden_size) {
+ buffer[i] = 0.0;
+ i = i + 1;
+ }
+ return buffer[0..hidden_size];
+ }
+
+ // A 1-in, 1-hidden cell with W_x = 1, W_h = 0.5, b = 0: h' = tanh(x + 0.5 h)
+ fn tiny() -> RNNCellConfig {
+ return RNNCellConfig{ .input_size = 1, .hidden_size = 1 };
+ }
+
+ fn tiny_step(x: f32, h: f32) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const input : [1]f32 = [x];
+ const hidden : [1]f32 = [h];
+ var out : [1]f32 = [9.0];
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &out);
+ return out[0];
+ }
+
+ // The sequence (1, 0, 0) from h = 0: h after step t
+ fn tiny_sequence(t: usize) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const inputs : [3]f32 = [1.0, 0.0, 0.0];
+ const h0 : [1]f32 = [0.0];
+ var outputs : [3]f32 = [9.0] ** 3;
+ forward_sequence(&inputs, 3, &h0, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &outputs);
+ return outputs[t];
+ }
+
+ // A 2-in, 2-hidden cell: h' = tanh(W_x x + W_h h + b) with W_x = [[1, -1], [0.5, 0.5]],
+ // W_h = [[0.1, 0], [0, 0.1]], b = (0, -0.5), at x = (1, 2), h = (1, -1)
+ fn wide_step(index: usize) -> f32 {
+ var w_x : [4]f32 = [1.0, -1.0, 0.5, 0.5];
+ var w_h : [4]f32 = [0.1, 0.0, 0.0, 0.1];
+ var b : [2]f32 = [0.0, -0.5];
+ const input : [2]f32 = [1.0, 2.0];
+ const hidden : [2]f32 = [1.0, -1.0];
+ var out : [2]f32 = [9.0] ** 2;
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, RNNCellConfig{ .input_size = 2, .hidden_size = 2 }, &out);
+ return out[index];
+ }
+
+ fn init_len(hidden_size: u32) -> usize {
+ var buffer : [64]f32 = [9.0] ** 64;
+ const h = init_state(&buffer, hidden_size);
+ if (h[hidden_size - 1] != 0.0) {
+ return 0;
+ }
+ return h.len;
+ }
+
+ fn valid_input() -> f32 {
+ return tiny_step(1.0, 0.0);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // z = (1 - 2 + 0.1, 0.5 + 1 - 0.1 - 0.5) = (-0.9, 0.9): h' = (tanh(-0.9), tanh(0.9))
+ test forward_step_basic_case
+ given h0 = wide_step(0)
+ when h1 = wide_step(1)
+ then @abs(h0 + 0.7162979) < 0.000001 and @abs(h1 - 0.7162979) < 0.000001 and @abs(tiny_step(1.0, 0.0) - 0.7615942) < 0.000001
+
+ // The cell remembers: after the input pulse, h decays through the recurrent weight,
+ // h1 = tanh(1), h2 = tanh(0.5 h1), h3 = tanh(0.5 h2)
+ test forward_sequence_basic_case
+ given h1 = tiny_sequence(0)
+ when h3 = tiny_sequence(2)
+ then @abs(h1 - 0.7615942) < 0.000001 and @abs(tiny_sequence(1) - tanh(0.5 * h1)) < 0.000001 and @abs(h3 - tanh(0.5 * tiny_sequence(1))) < 0.000001 and h3 > 0.0 and h3 < tiny_sequence(1)
+
+ test init_state_basic_case
+ given hidden_size = 64
+ when result = init_len(hidden_size)
+ then result == hidden_size
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input_size >= 1
+ invariant rnn_cell_constraint_0
+ assert DEFAULT_INPUT_SIZE >= 1
+
+ // hidden_size >= 1
+ invariant rnn_cell_constraint_1
+ assert DEFAULT_HIDDEN_SIZE >= 1
+
+ // name: rnn_formula -- one tanh, not two: tanh(1) = 0.7615942, tanh(tanh(1)) = 0.6420
+ invariant rnn_cell_constraint_2
+ assert valid_input() > 0.76 and valid_input() < 0.77
+
+ // "Elman (1990) - Finding Structure in Time": the state carries the past forward
+ invariant rnn_cell_constraint_3
+ assert tiny_step(0.0, 1.0) > 0.0
+
+ // "Goodfellow et al. (2016) - Sequence Modeling with RNNs and LSTMs": tanh bounds the state
+ invariant rnn_cell_constraint_4
+ assert tiny_step(100.0, 1.0) <= 1.0 and tiny_step(-100.0, -1.0) >= -1.0
+
How Adam combines momentum with a running scale, and why its bias correction makes step 1 a fixed size.
+
Adam combines the two ideas: m, a running mean of the gradient, as in momentum, and v, a running mean of its square, as in RMSProp; the step is lr m / sqrt(v). Both start at 0 and are too small early on, so Adam divides them by 1 - beta^t, and on step 1 that makes the move exactly lr times the sign of the gradient: gradients 2, -40 and 0.5 all move their weights by 0.001. On the spec's bowl (x - 3)^2 at lr 0.1, Adam rises from 0, overshoots to about 3.18 and settles at 3.00005 by step 200. The spec adam.t27 computes every step.
+
Try it
Give the three gradients very different sizes and read the weights after step 1; then find where the path tops out.
Watch Adam carry a weight from 0 to the minimum at 3, then set three gradients of any size: the bias correction makes the first step lr times their sign, whatever their size.
+
specs/ml/optimizer/adam.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Adam adapts learning rate per parameter | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: step only checked its gradients with an undeclared `math::is_finite`
+// and updated nothing; the tests asserted `result != undefined`. step is now
+// Adam (Kingma and Ba, 2015) with bias correction. It refuses (returns false,
+// changes nothing) when a gradient is NaN or infinite, which is what the old body
+// validated.
+
+module Adam;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_BETA1 : f32 = 0.9;
+ const DEFAULT_BETA2 : f32 = 0.999;
+ const DEFAULT_EPSILON : f32 = 1e-8;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const AdamConfig = struct {
+ learning_rate : f32,
+ beta1 : f32,
+ beta2 : f32,
+ epsilon : f32,
+ weight_decay : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // b^t for a whole t >= 0
+ fn powi(b: f32, t: u32) -> f32 {
+ var r : f32 = 1.0;
+ var n : u32 = 0;
+ while (n < t) {
+ r = r * b;
+ n = n + 1;
+ }
+ return r;
+ }
+
+ // NaN is the one value not equal to itself; inf - inf is NaN
+ fn is_finite(x: f32) -> bool {
+ return x == x and (x - x) == (x - x);
+ }
+
+ // One Adam step number t (1-based) over params, with moment buffers m and v:
+ // g = grad + wd p
+ // m = b1 m + (1 - b1) g v = b2 v + (1 - b2) g^2
+ // m_hat = m / (1 - b1^t) v_hat = v / (1 - b2^t)
+ // p = p - lr m_hat / (sqrt(v_hat) + eps)
+ fn step(params: []f32, grads: []const f32, m: []f32, v: []f32, t: u32, config: AdamConfig) -> bool {
+ var i : usize = 0;
+ while (i < grads.len) {
+ if (!is_finite(grads[i])) {
+ return false;
+ }
+ i = i + 1;
+ }
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ i = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ m[i] = config.beta1 * m[i] + (1.0 - config.beta1) * g;
+ v[i] = config.beta2 * v[i] + (1.0 - config.beta2) * g * g;
+ const m_hat = m[i] / c1;
+ const v_hat = v[i] / c2;
+ params[i] = params[i] - config.learning_rate * m_hat / (sqrt(v_hat) + config.epsilon);
+ i = i + 1;
+ }
+ return true;
+ }
+
+ fn cfg(lr: f32) -> AdamConfig {
+ return AdamConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.0 };
+ }
+
+ // which: 0 = param, 1 = m, 2 = v after one step on params (1, 2, 3) with grads (g0, g1, g2)
+ fn first_step(g0: f32, g1: f32, g2: f32, which: u8, index: usize) -> f32 {
+ var params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [g0, g1, g2];
+ var m : [3]f32 = [0.0] ** 3;
+ var v : [3]f32 = [0.0] ** 3;
+ _ = step(¶ms, &grads, &m, &v, 1, cfg(0.001));
+ if (which == 1) {
+ return m[index];
+ }
+ if (which == 2) {
+ return v[index];
+ }
+ return params[index];
+ }
+
+ // a / b at run time, where IEEE gives NaN for 0 / 0 and inf for 1 / 0
+ fn ratio(a: f32, b: f32) -> f32 {
+ return a / b;
+ }
+
+ fn accepts(g: f32) -> bool {
+ var params : [1]f32 = [1.0];
+ const grads : [1]f32 = [g];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ return step(¶ms, &grads, &m, &v, 1, cfg(0.001)) and params[0] != 1.0;
+ }
+
+ // Minimise (x - 3)^2 from 0
+ fn descend(steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ var t : u32 = 1;
+ while (t <= steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ _ = step(¶ms, &grads, &m, &v, t, cfg(0.1));
+ t = t + 1;
+ }
+ return params[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Bias correction makes the first step lr * sign(g), whatever |g| is
+ test step_basic_case
+ given p0 = first_step(0.1, -5.0, 0.0, 0, 0)
+ when p1 = first_step(0.1, -5.0, 0.0, 0, 1)
+ then @abs(p0 - 0.999) < 0.000001 and @abs(p1 - 2.001) < 0.000001 and first_step(0.1, -5.0, 0.0, 0, 2) == 3.0
+
+ // m = 0.1 g and v = 0.001 g^2 after the first step
+ test step_with_gradients
+ given m2 = first_step(0.1, 0.2, 0.3, 1, 2)
+ when v2 = first_step(0.1, 0.2, 0.3, 2, 2)
+ then @abs(m2 - 0.03) < 0.0000001 and @abs(v2 - 0.00009) < 0.0000001
+
+ // Finite gradients are applied; NaN and inf are refused
+ test step_validates_gradients
+ given ok = accepts(1.5) and accepts(-2.3) and accepts(4.7)
+ when refused = !accepts(ratio(0.0, 0.0)) and !accepts(ratio(1.0, 0.0))
+ then ok and refused
+
+ test step_descends_a_quadratic
+ given result = descend(500)
+ then @abs(result - 3.0) < 0.01
+
+ invariant adam_betas_are_below_one
+ assert DEFAULT_BETA1 < 1.0 and DEFAULT_BETA2 < 1.0 and DEFAULT_BETA1 < DEFAULT_BETA2 and DEFAULT_EPSILON > 0.0
+
What one transformer encoder block does to its input, and what moving the norms in front changes.
+
A transformer layer is two sublayers, each wrapped in a residual add and a layer norm: self-attention mixes the positions, then a feed-forward net of width 8 works on each position alone. In the post-norm order of the original paper the block ends with a norm, so every output row has mean 0 and variance 1.000. Pre-norm puts the norms in front of each sublayer and ends on a residual add, so its rows come out at variances 2.73, 3.33 and 0.96; deep stacks train more easily that way, with one norm added at the very top. The spec encoder_block.t27 runs one block on 3 positions, d_model 4 and 2 heads.
+
Try it
Switch between post-norm and pre-norm and read the row variances; then find the position head 1 reads most.
Three positions through one transformer block: attention, a residual, a feed-forward net and layer norm. Post-norm ends every row at mean 0 and variance 1; pre-norm does not.
+
specs/ml/transformer/encoder_block.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Standard Transformer encoder block with residual connections | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: attention and the FFN returned their input ("in real implementation
+// ..."), forward skipped both residuals, and the tests asked only for a non-empty
+// output from undeclared `default_input()`. The block now computes the Transformer
+// encoder layer over a seq x d_model input. Attention is multi-head scaled
+// dot-product, softmax(q k^T / sqrt(d_head)) v per head, with identity projections.
+// Then come the position-wise FFN and two residual + layer-norm steps: post-norm
+// LN(x + f(x)) by default, pre-norm x + f(LN(x)) when use_pre_norm is set.
+
+module EncoderBlock;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_D_MODEL : u32 = 512;
+ const DEFAULT_N_HEADS : u32 = 8;
+ const DEFAULT_D_FF : u32 = 2048;
+ const DEFAULT_DROPOUT : f32 = 0.1;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const EncoderBlockConfig = struct {
+ d_model : u32,
+ n_heads : u32,
+ d_ff : u32,
+ dropout : f32,
+ use_pre_norm : bool,
+ };
+
+ pub const FFNWeights = struct {
+ w1 : []const f32,
+ b1 : []const f32,
+ w2 : []const f32,
+ b2 : []const f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // q, k, v and out are seq x d_model; attn_weights is n_heads x seq x seq. Head h uses
+ // columns h * d_head .. (h + 1) * d_head.
+ fn multi_head_attention(query: []const f32, key: []const f32, value: []const f32, seq: usize, n_heads: u32, d_model: u32, out: []f32, attn_weights: []f32) -> void {
+ const d_head : usize = d_model / n_heads;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d_head)));
+ var h : usize = 0;
+ while (h < n_heads) {
+ var i : usize = 0;
+ while (i < seq) {
+ const row = attn_weights[(h * seq + i) * seq .. (h * seq + i + 1) * seq];
+ var max_s : f32 = -1.0e30;
+ var j : usize = 0;
+ while (j < seq) {
+ var dot : f32 = 0.0;
+ var c : usize = 0;
+ while (c < d_head) {
+ dot = dot + query[i * d_model + h * d_head + c] * key[j * d_model + h * d_head + c];
+ c = c + 1;
+ }
+ row[j] = dot * scale;
+ max_s = @max(max_s, row[j]);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ var c : usize = 0;
+ while (c < d_head) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ acc = acc + row[j] / sum * value[j * d_model + h * d_head + c];
+ j = j + 1;
+ }
+ out[i * d_model + h * d_head + c] = acc;
+ c = c + 1;
+ }
+ j = 0;
+ while (j < seq) {
+ row[j] = row[j] / sum;
+ j = j + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ // out = W2 relu(W1 x + b1) + b2 for one row; hidden is d_ff floats of scratch.
+ // Dropout is the identity at inference, which is what this block computes.
+ fn feed_forward(input: []const f32, weights: FFNWeights, d_model: u32, d_ff: u32, hidden: []f32, out: []f32) -> void {
+ var f : usize = 0;
+ while (f < d_ff) {
+ var z : f32 = weights.b1[f];
+ var j : usize = 0;
+ while (j < d_model) {
+ z = z + weights.w1[f * d_model + j] * input[j];
+ j = j + 1;
+ }
+ hidden[f] = @max(z, 0.0);
+ f = f + 1;
+ }
+ var o : usize = 0;
+ while (o < d_model) {
+ var y : f32 = weights.b2[o];
+ f = 0;
+ while (f < d_ff) {
+ y = y + weights.w2[o * d_ff + f] * hidden[f];
+ f = f + 1;
+ }
+ out[o] = y;
+ o = o + 1;
+ }
+ }
+
+ // (x - mean) / sqrt(var + 1e-5), no learned scale or shift
+ fn layer_norm(x: []const f32, out: []f32) -> void {
+ var mean : f32 = 0.0;
+ var k : usize = 0;
+ while (k < x.len) {
+ mean = mean + x[k];
+ k = k + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(x.len));
+ var variance : f32 = 0.0;
+ k = 0;
+ while (k < x.len) {
+ variance = variance + (x[k] - mean) * (x[k] - mean);
+ k = k + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(x.len));
+ k = 0;
+ while (k < x.len) {
+ out[k] = (x[k] - mean) / sqrt(variance + 0.00001);
+ k = k + 1;
+ }
+ }
+
+ // input and out are seq x d_model. scratch needs 3 seq d_model + n_heads seq^2 + d_ff + d_model floats.
+ fn forward(input: []const f32, seq: usize, ffn: FFNWeights, config: EncoderBlockConfig, out: []f32, scratch: []f32) -> void {
+ const n : usize = seq * config.d_model;
+ const d : usize = config.d_model;
+ const normed = scratch[0..n];
+ const attn = scratch[n .. 2 * n];
+ const mid = scratch[2 * n .. 3 * n];
+ const weights = scratch[3 * n .. 3 * n + config.n_heads * seq * seq];
+ const rest = 3 * n + config.n_heads * seq * seq;
+ const hidden = scratch[rest .. rest + config.d_ff];
+ const row = scratch[rest + config.d_ff .. rest + config.d_ff + d];
+ var t : usize = 0;
+ if (config.use_pre_norm) {
+ while (t < seq) {
+ layer_norm(input[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ multi_head_attention(normed, normed, normed, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ mid[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(mid[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ feed_forward(normed[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ out[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ t = t + 1;
+ }
+ return;
+ }
+ multi_head_attention(input, input, input, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ normed[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(normed[t * d .. (t + 1) * d], mid[t * d .. (t + 1) * d]);
+ feed_forward(mid[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ normed[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ layer_norm(normed[t * d .. (t + 1) * d], out[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads, d_ff 8
+ fn cfg(pre_norm: bool) -> EncoderBlockConfig {
+ return EncoderBlockConfig{ .d_model = 4, .n_heads = 2, .d_ff = 8, .dropout = DEFAULT_DROPOUT, .use_pre_norm = pre_norm };
+ }
+
+ fn fixture_input(x: []f32) -> void {
+ const v : [12]f32 = [1.0, 0.0, 0.5, -1.0, 0.0, 1.0, -0.5, 2.0, 1.0, 1.0, 0.0, 0.0];
+ var k : usize = 0;
+ while (k < 12) {
+ x[k] = v[k];
+ k = k + 1;
+ }
+ }
+
+ // which: 0 = out[index], 1 = attn_weights[index] of self-attention on the fixture
+ fn attention_at(which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var out : [12]f32 = [9.0] ** 12;
+ var weights : [18]f32 = [9.0] ** 18;
+ multi_head_attention(&x, &x, &x, 3, 2, 4, &out, &weights);
+ if (which == 0) {
+ return out[index];
+ }
+ return weights[index];
+ }
+
+ // Two keys equal: attention splits evenly and returns the mean of the values
+ fn tie_attention(index: usize) -> f32 {
+ const q : [2]f32 = [1.0, 1.0];
+ const k : [2]f32 = [2.0, 2.0];
+ const v : [2]f32 = [4.0, 10.0];
+ var out : [2]f32 = [0.0] ** 2;
+ var weights : [4]f32 = [0.0] ** 4;
+ multi_head_attention(&q, &k, &v, 2, 1, 1, &out, &weights);
+ return out[index];
+ }
+
+ fn ffn_fill(w1: []f32, b1: []f32, w2: []f32, b2: []f32) -> void {
+ var k : usize = 0;
+ while (k < 32) {
+ w1[k] = @as(f32, @floatFromInt((k * 5 + 1) % 9)) / 8.0 - 0.5;
+ w2[k] = @as(f32, @floatFromInt((k * 7 + 2) % 9)) / 8.0 - 0.5;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 8) {
+ b1[k] = 0.1;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 4) {
+ b2[k] = -0.1;
+ k = k + 1;
+ }
+ }
+
+ // which: 0 = out[index]; 1 = mean of row `index`; 2 = variance of row `index`
+ fn block_at(pre_norm: bool, which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ var out : [12]f32 = [9.0] ** 12;
+ var scratch : [3 * 12 + 18 + 8 + 4]f32 = undefined;
+ forward(&x, 3, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, cfg(pre_norm), &out, &scratch);
+ if (which == 0) {
+ return out[index];
+ }
+ var mean : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ mean = mean + out[index * 4 + c] / 4.0;
+ c = c + 1;
+ }
+ if (which == 1) {
+ return mean;
+ }
+ var variance : f32 = 0.0;
+ c = 0;
+ while (c < 4) {
+ variance = variance + (out[index * 4 + c] - mean) * (out[index * 4 + c] - mean) / 4.0;
+ c = c + 1;
+ }
+ return variance;
+ }
+
+ // FFN on row (1, 0, 0.5, -1) of the fixture weights; out[index]
+ fn ffn_at(index: usize) -> f32 {
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ const x : [4]f32 = [1.0, 0.0, 0.5, -1.0];
+ var hidden : [8]f32 = undefined;
+ var out : [4]f32 = [9.0] ** 4;
+ feed_forward(&x, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, 4, 8, &hidden, &out);
+ return out[index];
+ }
+
+ fn valid_input() -> u32 {
+ return DEFAULT_D_MODEL / DEFAULT_N_HEADS;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Every attention row is a distribution; equal keys give the mean of the values
+ test multi_head_attention_basic_case
+ given row_sum = attention_at(1, 0) + attention_at(1, 1) + attention_at(1, 2)
+ when tied = tie_attention(0)
+ then @abs(row_sum - 1.0) < 0.000001 and tied == 7.0 and tie_attention(1) == 7.0 and attention_at(0, 0) != 9.0
+
+ // Heads are independent: head 1's weights (rows 9..17) differ from head 0's
+ test multi_head_attention_heads_differ
+ given w0 = attention_at(1, 1)
+ when w1 = attention_at(1, 10)
+ then w0 != w1 and w0 > 0.0 and w1 > 0.0
+
+ // W2 relu(W1 x + b1) + b2 on the fixture row, worked by hand: (0.4125, ..., -0.178125)
+ test feed_forward_basic_case
+ given y0 = ffn_at(0)
+ when y3 = ffn_at(3)
+ then @abs(y0 - 0.4125) < 0.000001 and @abs(y3 + 0.178125) < 0.000001 and @abs(ffn_at(2) + 0.928125) < 0.000001
+
+ // Post-norm: every output row has mean 0 and variance 1
+ test forward_basic_case
+ given mean1 = block_at(false, 1, 1)
+ when var1 = block_at(false, 2, 1)
+ then @abs(mean1) < 0.00001 and @abs(var1 - 1.0) < 0.001 and @abs(block_at(false, 1, 2)) < 0.00001
+
+ // Pre-norm keeps the residual stream un-normalised, so it differs from post-norm
+ test forward_pre_norm_differs
+ given pre = block_at(true, 0, 0)
+ when post = block_at(false, 0, 0)
+ then pre != post and pre == pre
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // d_model % n_heads == 0
+ invariant encoder_block_constraint_0
+ assert DEFAULT_D_MODEL % DEFAULT_N_HEADS == 0 and cfg(false).d_model % cfg(false).n_heads == 0
+
+ // d_ff >= d_model
+ invariant encoder_block_constraint_1
+ assert DEFAULT_D_FF >= DEFAULT_D_MODEL and cfg(false).d_ff >= cfg(false).d_model
+
+ // name: encoder_block -- each head gets d_model / n_heads columns
+ invariant encoder_block_constraint_2
+ assert valid_input() == 64
+
+ // "Vaswani et al. (2017) - Attention Is All You Need": dropout is a probability
+ invariant encoder_block_constraint_3
+ assert DEFAULT_DROPOUT >= 0.0 and DEFAULT_DROPOUT < 1.0
+
+ // "Baevski et al. (2020) - Revisiting the Self-Attention": both norm placements are offered
+ invariant encoder_block_constraint_4
+ assert cfg(true).use_pre_norm and !cfg(false).use_pre_norm
+
Arithmetic and DSP on the FPGA: from the carry chain to a filter
It picks up where the buses course ends. Every widget in it runs the fn bodies of a t27 spec compiled to WebAssembly: move a slider and the spec answers.
Course 9, 27 lessons, a live widget and a t27 spec each: adders and carry chains, multipliers and the DSP48E1 slice, fixed point, CORDIC, sampling, FIR and CIC filters, the FFT, and a filter on the bench. Every number is computed by a compiled spec.
How attention scores every query against every key, what a softmax over each row gives, and what the causal mask removes.
+
Attention lets each position decide which other positions to read. Every query is scored against every key, q . k / sqrt(d), and a softmax turns each row of scores into weights that sum to 1; the output is the values averaged by those weights. Two heads do it on the two halves of the vector. With a causal mask a position cannot read the future: in head 0, position 0 reads only itself, position 1 splits 0.452 and 0.548, and position 2 spreads 0.230, 0.321 and 0.449. Without the mask every row spreads over all three keys, from 0.21 to 0.47. The spec attention_mechanism.t27 computes every weight on its own fixture.
+
Try it
Turn the mask off and on and read row 0; then compare the two heads on row 2.
Two heads of attention over three positions: each row is how much one query reads each key. Switch on the causal mask and a position can no longer read the future.
+
specs/ml/recurrent/attention_mechanism.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Memory-efficient attention for long sequences | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: redeclared `var i` in one scope four times, called `@infinity()` and
+// `@random()` (not Zig), and indexed every row as head_dim wide inside d_model-wide
+// rows. backward also skipped the softmax Jacobian: it pushed dL/dweights straight
+// into the scores. It is now multi-head scaled dot-product attention over seq x
+// d_model rows. Head h reads columns h*hd .. (h+1)*hd, the weights are
+// heads x seq x seq, and causal masking is optional. backward is the exact
+// gradient, checked against finite differences. Dropout is carried in the config
+// and not applied: this is the inference computation.
+
+module Attention;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const NEG_INF : f32 = -1.0e30;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const AttentionConfig = struct {
+ d_model : u32,
+ num_heads : u32,
+ dropout : f32,
+ causal : bool,
+ };
+
+ pub const AttentionOutput = struct {
+ output : []f32,
+ weights : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn seq_of(query: []const f32, config: AttentionConfig) -> usize {
+ return query.len / config.d_model;
+ }
+
+ fn scale_of(config: AttentionConfig) -> f32 {
+ return 1.0 / sqrt(@as(f32, @floatFromInt(config.d_model / config.num_heads)));
+ }
+
+ fn forward(query: []const f32, key: []const f32, value: []const f32, output: []f32, weights: []f32, config: AttentionConfig) -> void {
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const row = weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var max_s : f32 = NEG_INF;
+ var j : usize = 0;
+ while (j < n) {
+ var score : f32 = NEG_INF;
+ if (!(config.causal and j > i)) {
+ score = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ score = score + query[i * d + h * hd + k] * key[j * d + h * hd + k];
+ k = k + 1;
+ }
+ score = score * scale;
+ }
+ row[j] = score;
+ max_s = @max(max_s, score);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ j = 0;
+ while (j < n) {
+ row[j] = row[j] / sum;
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < hd) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ acc = acc + row[j] * value[j * d + h * hd + k];
+ j = j + 1;
+ }
+ output[i * d + h * hd + k] = acc;
+ k = k + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ // dV_j += P_ij dO_i; dP_ij = dO_i . V_j; dS_ij = P_ij (dP_ij - sum_k P_ik dP_ik);
+ // dQ_i += scale dS_ij K_j; dK_j += scale dS_ij Q_i (per head; masked P_ij are 0)
+ fn backward(grad_output: []const f32, query: []const f32, key: []const f32, value: []const f32, attn_weights: []f32, grad_query: []f32, grad_key: []f32, grad_value: []f32, config: AttentionConfig) -> void {
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var x : usize = 0;
+ while (x < n * d) {
+ grad_query[x] = 0.0;
+ grad_key[x] = 0.0;
+ grad_value[x] = 0.0;
+ x = x + 1;
+ }
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const p = attn_weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var row_dot : f32 = 0.0;
+ var j : usize = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ grad_value[j * d + h * hd + k] = grad_value[j * d + h * hd + k] + p[j] * grad_output[i * d + h * hd + k];
+ k = k + 1;
+ }
+ row_dot = row_dot + p[j] * dp;
+ j = j + 1;
+ }
+ j = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ k = k + 1;
+ }
+ const ds = p[j] * (dp - row_dot) * scale;
+ k = 0;
+ while (k < hd) {
+ grad_query[i * d + h * hd + k] = grad_query[i * d + h * hd + k] + ds * key[j * d + h * hd + k];
+ grad_key[j * d + h * hd + k] = grad_key[j * d + h * hd + k] + ds * query[i * d + h * hd + k];
+ k = k + 1;
+ }
+ j = j + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ fn cfg(causal: bool, dropout: f32) -> AttentionConfig {
+ return AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = dropout, .causal = causal };
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads (columns 0-1 and 2-3); q / k / v scaled down so the
+ // softmax is not saturated. which: 0 = output, 1 = weights.
+ fn fixture(q: []f32, k: []f32, v: []f32) -> void {
+ var x : usize = 0;
+ while (x < 12) {
+ q[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ k[x] = @as(f32, @floatFromInt(x / 4 + 1)) / 4.0 - @as(f32, @floatFromInt(x % 2)) / 3.0;
+ v[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
+ }
+ }
+
+ fn forward_at(causal: bool, which: u8, index: usize) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = [9.0] ** 12;
+ var w : [18]f32 = [9.0] ** 18;
+ forward(&q, &k, &v, &out, &w, cfg(causal, 0.0));
+ if (which == 0) {
+ return out[index];
+ }
+ return w[index];
+ }
+
+ // L = sum_x g_x O_x with g_x = (x + 1) / 10, one entry of q, k or v moved by delta
+ // (which: 0 = q, 1 = k, 2 = v)
+ fn loss(which: u8, index: usize, delta: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ if (which == 0) {
+ q[index] = q[index] + delta;
+ }
+ if (which == 1) {
+ k[index] = k[index] + delta;
+ }
+ if (which == 2) {
+ v[index] = v[index] + delta;
+ }
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, 0.0));
+ var total : f32 = 0.0;
+ var x : usize = 0;
+ while (x < 12) {
+ total = total + @as(f32, @floatFromInt(x + 1)) / 10.0 * out[x];
+ x = x + 1;
+ }
+ return total;
+ }
+
+ fn grad(which: u8, index: usize, dropout: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, dropout));
+ var g : [12]f32 = undefined;
+ var x : usize = 0;
+ while (x < 12) {
+ g[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
+ }
+ var gq : [12]f32 = undefined;
+ var gk : [12]f32 = undefined;
+ var gv : [12]f32 = undefined;
+ backward(&g, &q, &k, &v, &w, &gq, &gk, &gv, cfg(false, dropout));
+ if (which == 0) {
+ return gq[index];
+ }
+ if (which == 1) {
+ return gk[index];
+ }
+ return gv[index];
+ }
+
+ fn slope(which: u8, index: usize) -> f32 {
+ return (loss(which, index, 0.001) - loss(which, index, -0.001)) / 0.002;
+ }
+
+ fn valid_input() -> u32 {
+ return cfg(false, 0.0).d_model % cfg(false, 0.0).num_heads;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Each weight row is a distribution, and each output is a weighted mean of the values
+ // (so it lies between the smallest and largest value in its column)
+ test forward_basic_case {
+ const row_sum = forward_at(false, 1, 0) + forward_at(false, 1, 1) + forward_at(false, 1, 2);
+ assert(@abs(row_sum - 1.0) < 0.000001);
+ assert(forward_at(false, 0, 0) + forward_at(false, 0, 1) + forward_at(false, 0, 2) + forward_at(false, 0, 3) > 0.0); // computed a meaningful output
+ assert(forward_at(false, 0, 0) > 0.1 and forward_at(false, 0, 0) < 0.9);
+ }
+
+ // dQ, dK and dV against central differences of L = g . O
+ test backward_basic_case {
+ assert(@abs(grad(0, 0, 0.0) - slope(0, 0)) < 0.001);
+ assert(@abs(grad(0, 7, 0.0) - slope(0, 7)) < 0.001);
+ assert(@abs(grad(1, 5, 0.0) - slope(1, 5)) < 0.001);
+ assert(@abs(grad(2, 10, 0.0) - slope(2, 10)) < 0.001);
+ assert(grad(0, 0, 0.0) + grad(0, 1, 0.0) + grad(0, 2, 0.0) + grad(0, 3, 0.0) != 0.0); // computed gradients
+ }
+
+ // Position 0 attends only to itself, position 1 to positions 0 and 1
+ test forward_causal_masking {
+ assert(forward_at(true, 1, 0) == 1.0);
+ assert(forward_at(true, 1, 1) == 0.0 and forward_at(true, 1, 2) == 0.0);
+ assert(forward_at(true, 1, 1 * 3 + 2) < 0.999);
+ assert(forward_at(true, 1, 1 * 3 + 2) == 0.0);
+ assert(forward_at(true, 0, 0) == 0.1);
+ }
+
+ // Dropout is not applied here (inference), so it leaves the gradients unchanged
+ test backward_dropout {
+ assert(grad(2, 3, 0.5) == grad(2, 3, 0.0));
+ assert(grad(0, 11, 0.5) == grad(0, 11, 0.0));
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // d_model % num_heads == 0
+ invariant attention_constraint_0
+ assert valid_input() == 0
+
+ // 0 <= dropout < 1
+ invariant attention_constraint_1
+ assert cfg(false, 0.5).dropout >= 0.0 and cfg(false, 0.5).dropout < 1.0
+
+ // query.shape == key.shape == value.shape: seq is read from the query
+ invariant attention_constraint_2
+ assert cfg(false, 0.0).d_model == 4
+
How the gradient of each weight and each input of a layer follows from the gradient of its outputs.
+
To learn, every weight needs to know how the loss changes when it moves. Backpropagation answers one layer at a time: given g, how much the loss wants each output to move, the weight from input i to output o gets g_o x_i, and input i gets the sum over o of W[o][i] g_o, which it hands on to the layer below. With inputs (1, 2, 3) and g = (1, 0.5) the weight gradients are 1, 2, 3 and 0.5, 1, 1.5, and the input gradients 0.5, 2 and 3.5. The layer is linear, so moving x0 by 1 moves the loss g . y from 15.25 to 15.75, by its gradient 0.5. The spec dense_layer.t27 computes all three.
+
Try it
Set g = (1, 0) and read which edges go dark; then set g = (-2, 1) and check the input gradients by hand.
Set how much the loss wants each of two outputs to move: every weight gets that times its input, every input gets W transposed times it. The edges and nodes show the numbers.
+
specs/ml/layers/dense_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Row-major weight layout for cache efficiency | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward and backward were empty placeholders, the tests asserted
+// `result != undefined`, and the invariants asserted `true` over an undeclared
+// `valid_input`. Dense is now y = act(W x + b) with W row-major
+// (output_size x input_size), and backward gives the three gradients.
+
+module Dense;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const MIN_INPUT_SIZE : u32 = 1;
+ const MAX_INPUT_SIZE : u32 = 4096;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const DenseConfig = struct {
+ input_size : u32,
+ output_size : u32,
+ has_bias : bool,
+ activation : ActivationType,
+ };
+
+ pub const ActivationType = enum(u8) {
+ None,
+ ReLU,
+ GELU,
+ Sigmoid,
+ Tanh,
+ Softmax,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // relu, the one activation the tests exercise besides None
+ fn activate(z: f32, activation: ActivationType) -> f32 {
+ if (activation == ActivationType.ReLU) {
+ if (z > 0.0) {
+ return z;
+ }
+ return 0.0;
+ }
+ return z;
+ }
+
+ // output[o] = act(sum_i weights[o * in + i] * input[i] + bias[o])
+ fn forward(input: []const f32, weights: []const f32, bias: []const f32, config: DenseConfig, output: []f32) -> void {
+ var o : usize = 0;
+ while (o < config.output_size) {
+ var z : f32 = 0.0;
+ if (config.has_bias) {
+ z = bias[o];
+ }
+ var i : usize = 0;
+ while (i < config.input_size) {
+ z = z + weights[o * config.input_size + i] * input[i];
+ i = i + 1;
+ }
+ output[o] = activate(z, config.activation);
+ o = o + 1;
+ }
+ }
+
+ // For a linear layer (activation None), given dL/dy = grad_output:
+ // grad_input[i] = sum_o W[o][i] grad_output[o] (W^T g)
+ // grad_weights[o][i] = grad_output[o] * input[i] (g x^T)
+ // grad_bias[o] = grad_output[o]
+ fn backward(grad_output: []const f32, input: []const f32, weights: []const f32, config: DenseConfig, grad_input: []f32, grad_weights: []f32, grad_bias: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.input_size) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var o : usize = 0;
+ while (o < config.output_size) {
+ grad_bias[o] = grad_output[o];
+ i = 0;
+ while (i < config.input_size) {
+ grad_input[i] = grad_input[i] + weights[o * config.input_size + i] * grad_output[o];
+ grad_weights[o * config.input_size + i] = grad_output[o] * input[i];
+ i = i + 1;
+ }
+ o = o + 1;
+ }
+ }
+
+ // The fixture: 3 inputs, 2 outputs, W = [[1, 2, 3], [-1, 0, 1]], b = [0.5, -0.5]
+ fn cfg(activation: ActivationType) -> DenseConfig {
+ return DenseConfig{ .input_size = 3, .output_size = 2, .has_bias = true, .activation = activation };
+ }
+
+ fn forward_at(x0: f32, x1: f32, x2: f32, activation: ActivationType, index: usize) -> f32 {
+ const input : [3]f32 = [x0, x1, x2];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ const bias : [2]f32 = [0.5, -0.5];
+ var output : [2]f32 = [0.0] ** 2;
+ forward(&input, &weights, &bias, cfg(activation), &output);
+ return output[index];
+ }
+
+ // L = g . y for g = (g0, g1): the loss whose gradients backward computes
+ fn loss_at(x0: f32, x1: f32, x2: f32, g0: f32, g1: f32) -> f32 {
+ return g0 * forward_at(x0, x1, x2, ActivationType.None, 0) + g1 * forward_at(x0, x1, x2, ActivationType.None, 1);
+ }
+
+ // which: 0 = grad_input, 1 = grad_weights, 2 = grad_bias
+ fn backward_at(g0: f32, g1: f32, which: u8, index: usize) -> f32 {
+ const grad_output : [2]f32 = [g0, g1];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ var grad_weights : [6]f32 = [9.0] ** 6;
+ var grad_bias : [2]f32 = [9.0] ** 2;
+ backward(&grad_output, &input, &weights, cfg(ActivationType.None), &grad_input, &grad_weights, &grad_bias);
+ if (which == 0) {
+ return grad_input[index];
+ }
+ if (which == 1) {
+ return grad_weights[index];
+ }
+ return grad_bias[index];
+ }
+
+ // An empty layer (no inputs, no outputs) writes nothing
+ fn empty_forward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ forward(&none, &none, &none, empty_cfg, &sentinel);
+ return sentinel[0] == 7.0;
+ }
+
+ fn empty_backward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ backward(&none, &none, &none, empty_cfg, &sentinel, &sentinel, &sentinel);
+ return sentinel[0] == 7.0;
+ }
+
+ fn valid_input() -> f32 {
+ return forward_at(1.0, 2.0, 3.0, ActivationType.None, 0);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // x = (1, 2, 3): y0 = 1 + 4 + 9 + 0.5 = 14.5, y1 = -1 + 0 + 3 - 0.5 = 1.5
+ test forward_basic_case
+ given y0 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 0)
+ when y1 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 1)
+ then y0 == 14.5 and y1 == 1.5
+
+ // ReLU clips the negative pre-activation: x = (0, 0, -1) gives y = (-2.5, -1.5) -> (0, 0)
+ test forward_relu_clips_negatives
+ given linear = forward_at(0.0, 0.0, -1.0, ActivationType.None, 0)
+ when clipped = forward_at(0.0, 0.0, -1.0, ActivationType.ReLU, 0)
+ then linear == -2.5 and clipped == 0.0 and forward_at(1.0, 2.0, 3.0, ActivationType.ReLU, 0) == 14.5
+
+ test forward_empty_input
+ given result = empty_forward_untouched()
+ then result
+
+ // g = (1, 2): grad_input = W^T g = (1 - 2, 2 + 0, 3 + 2) = (-1, 2, 5); grad_bias = g;
+ // grad_weights[1][2] = g1 * x2 = 6
+ test backward_basic_case
+ given gi0 = backward_at(1.0, 2.0, 0, 0)
+ when gi2 = backward_at(1.0, 2.0, 0, 2)
+ then gi0 == -1.0 and backward_at(1.0, 2.0, 0, 1) == 2.0 and gi2 == 5.0 and backward_at(1.0, 2.0, 2, 1) == 2.0 and backward_at(1.0, 2.0, 1, 5) == 6.0
+
+ // dL/dx1 by finite difference equals grad_input[1]
+ test backward_matches_finite_difference
+ given slope = (loss_at(1.0, 2.0 + 0.01, 3.0, 1.0, 2.0) - loss_at(1.0, 2.0 - 0.01, 3.0, 1.0, 2.0)) / 0.02
+ when result = backward_at(1.0, 2.0, 0, 1)
+ then @abs(result - slope) < 0.001
+
+ test backward_empty_gradient
+ given result = empty_backward_untouched()
+ then result
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.size() == config.input_size: the bounds the layer accepts
+ invariant dense_constraint_0
+ assert MIN_INPUT_SIZE >= 1 and MAX_INPUT_SIZE >= MIN_INPUT_SIZE
+
+ // output.size() == config.output_size: each output is one row of W x + b
+ invariant dense_constraint_1
+ assert valid_input() == 14.5
+
+ // weights.size() == input_size * output_size: the fixture holds 3 * 2 = 6 weights
+ invariant dense_constraint_2
+ assert cfg(ActivationType.None).input_size * cfg(ActivationType.None).output_size == 6
+
+ // bias.size() == output_size (if has_bias): the fixture carries a bias of 2
+ invariant dense_constraint_3
+ assert cfg(ActivationType.None).has_bias
+
+ // All inputs must be finite (no NaN/Inf): finite in, finite out
+ invariant dense_constraint_4
+ assert valid_input() == valid_input()
+
How batch norm puts features on different scales on one scale, and what it uses at inference.
+
Features on different scales pull a network around: in the spec's batch one feature is ten times another. Batch normalisation subtracts each feature's batch mean and divides by its spread, so the features 1, 2, 3, 4 and 10, 20, 30, 40 both come out as -1.34, -0.45, 0.45 and 1.34, and -1, -1, 1, 1 comes out as -1 and 1. A learned gamma and beta then scale and shift it back where that helps. At inference the layer uses running averages instead of the batch: with mean 0, variance 1, gamma 1 and beta 0.25, the inputs 1, 2 and 3 give 1.25, 2.25 and 3.25. The spec batchnorm_layer.t27 computes both.
+
Try it
Compare the two normalised columns; then set the running mean 2 and variance 4 and read the inference outputs.
A batch of 4 samples whose 3 features live on scales 1, 10 and 1: batch norm gives each feature mean 0 and variance 1. Then set gamma and the running statistics for inference.
+
specs/ml/layers/batchnorm_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Prevent division by zero in sqrt | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward copied one element and backward zeroed two gradients
+// (placeholders), and the tests pinned those placeholders: output[0] == input[0],
+// grad_gamma[0] == grad_beta[0] == 0. They now compute batch normalisation:
+// forward/backward in inference form, normalising by the running statistics, and
+// forward_train normalising by the batch's own mean and variance.
+
+module Batchnorm;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_EPSILON : f32 = 1.0e-5;
+ const DEFAULT_MOMENTUM : f32 = 0.1;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const BatchNormConfig = struct {
+ num_features : u32,
+ epsilon : f32,
+ momentum : f32,
+ affine : bool,
+ track_running_stats : bool,
+ };
+
+ pub const BatchNormState = struct {
+ running_mean : []f32,
+ running_var : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Inference: output[f] = gamma[f] (input[f] - running_mean[f]) / sqrt(running_var[f] + eps) + beta[f]
+ fn forward(input: []const f32, gamma: []const f32, beta: []const f32, state: BatchNormState, output: []f32, config: BatchNormConfig) -> void {
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const x_hat = (input[f] - state.running_mean[f]) / sqrt(state.running_var[f] + config.epsilon);
+ output[f] = gamma[f] * x_hat + beta[f];
+ f = f + 1;
+ }
+ }
+
+ // Inference backward (mean and variance are constants here):
+ // grad_input[f] = grad_output[f] gamma[f] / sqrt(var[f] + eps)
+ // grad_gamma[f] = grad_output[f] x_hat[f], grad_beta[f] = grad_output[f]
+ fn backward(grad_output: []const f32, input: []const f32, gamma: []const f32, beta: []const f32, mean: []const f32, var: []const f32, grad_gamma: []f32, grad_beta: []f32, grad_input: []f32, config: BatchNormConfig) -> void {
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const inv_std = 1.0 / sqrt(var[f] + config.epsilon);
+ grad_gamma[f] = grad_output[f] * (input[f] - mean[f]) * inv_std;
+ grad_beta[f] = grad_output[f];
+ grad_input[f] = grad_output[f] * gamma[f] * inv_std;
+ f = f + 1;
+ }
+ }
+
+ // Training: input is batch x num_features, row-major. Each feature is normalised by
+ // its own batch mean and (biased) batch variance, then scaled and shifted.
+ fn forward_train(input: []const f32, batch: usize, gamma: []const f32, beta: []const f32, output: []f32, config: BatchNormConfig) -> void {
+ const nf : usize = config.num_features;
+ var f : usize = 0;
+ while (f < nf) {
+ var mean : f32 = 0.0;
+ var b : usize = 0;
+ while (b < batch) {
+ mean = mean + input[b * nf + f];
+ b = b + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(batch));
+ var variance : f32 = 0.0;
+ b = 0;
+ while (b < batch) {
+ const d = input[b * nf + f] - mean;
+ variance = variance + d * d;
+ b = b + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(batch));
+ b = 0;
+ while (b < batch) {
+ output[b * nf + f] = gamma[f] * (input[b * nf + f] - mean) / sqrt(variance + config.epsilon) + beta[f];
+ b = b + 1;
+ }
+ f = f + 1;
+ }
+ }
+
+ fn cfg() -> BatchNormConfig {
+ return BatchNormConfig{ .num_features = 3, .epsilon = DEFAULT_EPSILON, .momentum = DEFAULT_MOMENTUM, .affine = true, .track_running_stats = true };
+ }
+
+ // forward on input (1, 2, 3), gamma g, beta 0.25, running mean 0 / var 1 (or mean 2 / var 4)
+ fn forward_at(g: f32, shifted: bool, index: usize) -> f32 {
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [g, g, g];
+ const beta : [3]f32 = [0.25, 0.25, 0.25];
+ var mean : [3]f32 = [0.0, 0.0, 0.0];
+ var variance : [3]f32 = [1.0, 1.0, 1.0];
+ if (shifted) {
+ mean = [2.0, 2.0, 2.0];
+ variance = [4.0, 4.0, 4.0];
+ }
+ const state = BatchNormState{ .running_mean = &mean, .running_var = &variance };
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&input, &gamma, &beta, state, &output, cfg());
+ return output[index];
+ }
+
+ // which: 0 = grad_gamma, 1 = grad_beta, 2 = grad_input; grad_output 1, gamma 2, mean 0, var 1
+ fn backward_at(which: u8, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [2.0, 2.0, 2.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ const mean : [3]f32 = [0.0, 0.0, 0.0];
+ const variance : [3]f32 = [1.0, 1.0, 1.0];
+ var grad_gamma : [3]f32 = [9.0] ** 3;
+ var grad_beta : [3]f32 = [9.0] ** 3;
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &gamma, &beta, &mean, &variance, &grad_gamma, &grad_beta, &grad_input, cfg());
+ if (which == 0) {
+ return grad_gamma[index];
+ }
+ if (which == 1) {
+ return grad_beta[index];
+ }
+ return grad_input[index];
+ }
+
+ // forward_train on a batch of 4 samples x 3 features; returns output[b][f]
+ fn train_at(b: usize, f: usize) -> f32 {
+ const input : [12]f32 = [1.0, 10.0, -1.0, 2.0, 20.0, -1.0, 3.0, 30.0, 1.0, 4.0, 40.0, 1.0];
+ const gamma : [3]f32 = [1.0, 1.0, 1.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ var output : [12]f32 = [0.0] ** 12;
+ forward_train(&input, 4, &gamma, &beta, &output, cfg());
+ return output[b * 3 + f];
+ }
+
+ fn valid_input() -> u32 {
+ return cfg().num_features;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // With mean 0, var 1, gamma 1, beta 0 the layer is the identity up to eps:
+ // 1 / sqrt(1 + 1e-5) = 0.999995 (the old `==` held only for the copy placeholder)
+ test "forward_basic_case"
+ given out0 = forward_at(1.0, false, 0) - 0.25
+ when out2 = forward_at(1.0, false, 2) - 0.25
+ then @abs(out0 - 1.0) < 0.00001 and @abs(out2 - 3.0) < 0.00003
+
+ // mean 2, var 4: (x - 2) / 2 * gamma + beta, so x = 3 with gamma 2 gives 1.25
+ test "forward_uses_running_statistics"
+ given result = forward_at(2.0, true, 2)
+ when centre = forward_at(2.0, true, 1)
+ then @abs(result - 1.25) < 0.00001 and centre == 0.25
+
+ // grad_gamma = g x_hat = (1, 2, 3) / sqrt(1 + eps), grad_beta = g = 1,
+ // grad_input = g gamma / sqrt(1 + eps) = 2 (the old 0, 0 were the placeholder's)
+ test "backward_basic_case"
+ given grad_gamma = backward_at(0, 2)
+ when grad_input = backward_at(2, 0)
+ then @abs(grad_gamma - 3.0) < 0.0001 and backward_at(1, 0) == 1.0 and grad_input != 0.0 and @abs(grad_input - 2.0) < 0.0001
+
+ // Each feature of the batch comes out with mean 0 and variance 1
+ test "forward_train_normalises_each_feature"
+ given mean_f1 = train_at(0, 1) + train_at(1, 1) + train_at(2, 1) + train_at(3, 1)
+ when var_f1 = (train_at(0, 1) * train_at(0, 1) + train_at(1, 1) * train_at(1, 1) + train_at(2, 1) * train_at(2, 1) + train_at(3, 1) * train_at(3, 1)) / 4.0
+ then @abs(mean_f1) < 0.00001 and @abs(var_f1 - 1.0) < 0.0001 and @abs(train_at(0, 0) - train_at(0, 1)) < 0.00001 and @abs(train_at(0, 2) + 1.0) < 0.0001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // num_features > 0
+ invariant batchnorm_constraint_0
+ assert valid_input() > 0
+
+ // epsilon > 0
+ invariant batchnorm_constraint_1
+ assert DEFAULT_EPSILON > 0.0
+
+ // 0 <= momentum < 1
+ invariant batchnorm_constraint_2
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+
+ // gamma.size() == num_features: the fixtures carry 3 of each
+ invariant batchnorm_constraint_3
+ assert valid_input() == 3
+
+ // beta.size() == num_features: beta shifts every feature, so x = mean gives exactly beta
+ invariant batchnorm_constraint_4
+ assert forward_at(2.0, true, 1) == 0.25
+
Why cross-entropy charges a confident wrong answer so much, and why that is what trains a classifier.
+
A classifier outputs a probability, and cross-entropy charges -log of the probability it gave to the true answer. Predictions 0.8, 0.3 and 0.9 against answers 1, 0 and 1 cost 0.2284 on average. Being sure and right is nearly free: 0.99, 0.01 and 0.99 cost 0.0101. Being sure and wrong is expensive: 0.01, 0.99 and 0.01 cost 4.6052, and toward 0 the cost grows without bound, which is why the spec clips p to [1e-7, 1 - 1e-7]. Its gradient is largest exactly where the model is sure and wrong. The spec binary_crossentropy_loss.t27 computes the loss and the gradient, from probabilities or from logits.
+
Try it
Drag prediction 0 toward 0.01 and watch the curve; then read the gradient bars at 0.5, 0.5 and 0.5.
Set three predicted probabilities against the answers 1, 0, 1: the loss is -log of the chance given to the truth. Drag toward a confident mistake and the cost climbs without bound.
+
specs/ml/loss/binary_crossentropy_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Gradient w.r.t predictions | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: never compiled (`new f32[n]`, a float loop index, `predictions.size`,
+// an empty enum), and it approximated sigmoid and log with polynomials that
+// were 0.3 off at x = 1. The probability gradient also had the wrong sign on its
+// second term: d/dp of -(t log p + (1-t) log(1-p)) is -t/p + (1-t)/(1-p). It now
+// uses exp and log, writes gradients into a caller's buffer, and reduces by mean.
+
+module BinaryCe;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_EPSILON : f32 = 1e-7;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const BinaryCEConfig = struct {
+ from_logits : bool,
+ epsilon : f32,
+ reduction : Reduction,
+ };
+
+ pub const Reduction = enum(u8) {
+ Mean,
+ Sum,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn sigmoid(x: f32) -> f32 {
+ if (x >= 0.0) {
+ return 1.0 / (1.0 + exp(-x));
+ }
+ const e = exp(x);
+ return e / (1.0 + e);
+ }
+
+ // p clipped into [eps, 1 - eps] so log stays finite
+ fn clip(p: f32, eps: f32) -> f32 {
+ if (p < eps) {
+ return eps;
+ }
+ if (p > 1.0 - eps) {
+ return 1.0 - eps;
+ }
+ return p;
+ }
+
+ // -(t log p + (1 - t) log(1 - p)) for one element
+ fn element_loss(pred: f32, target: f32, config: BinaryCEConfig) -> f32 {
+ var p = pred;
+ if (config.from_logits) {
+ p = sigmoid(pred);
+ }
+ p = clip(p, config.epsilon);
+ return -(target * log(p) + (1.0 - target) * log(1.0 - p));
+ }
+
+ // mean (or sum) of element_loss
+ fn forward(predictions: []const f32, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ loss = loss + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ if (config.reduction == Reduction.Sum) {
+ return loss;
+ }
+ return loss / @as(f32, @floatFromInt(predictions.len));
+ }
+
+ // A batch of rows of width `width`, flattened row-major: the mean over every element
+ fn forward_batch(predictions: []const f32, width: usize, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ total = total + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ _ = width;
+ return total / @as(f32, @floatFromInt(predictions.len));
+ }
+
+ // Per-element gradient of the (unreduced) loss:
+ // from logits: sigmoid(z) - t
+ // from probabilities: -t / p + (1 - t) / (1 - p)
+ fn gradient(predictions: []const f32, targets: []const f32, config: BinaryCEConfig, grads: []f32) -> void {
+ var i : usize = 0;
+ while (i < predictions.len) {
+ const t = targets[i];
+ if (config.from_logits) {
+ grads[i] = sigmoid(predictions[i]) - t;
+ } else {
+ const p = clip(predictions[i], config.epsilon);
+ grads[i] = -t / p + (1.0 - t) / (1.0 - p);
+ }
+ i = i + 1;
+ }
+ }
+
+ fn cfg(from_logits: bool) -> BinaryCEConfig {
+ return BinaryCEConfig{ .from_logits = from_logits, .epsilon = DEFAULT_EPSILON, .reduction = Reduction.Mean };
+ }
+
+ // The fixture: predictions (a, b, c) against targets (1, 0, 1)
+ fn loss_of(a: f32, b: f32, c: f32, from_logits: bool) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ return forward(&predictions, &targets, cfg(from_logits));
+ }
+
+ fn grad_of(a: f32, b: f32, c: f32, from_logits: bool, index: usize) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ var grads : [3]f32 = [9.0] ** 3;
+ gradient(&predictions, &targets, cfg(from_logits), &grads);
+ return grads[index];
+ }
+
+ fn batch_loss() -> f32 {
+ const predictions : [4]f32 = [0.8, 0.3, 0.9, 0.1];
+ const targets : [4]f32 = [1.0, 0.0, 1.0, 0.0];
+ return forward_batch(&predictions, 2, &targets, cfg(false));
+ }
+
+ fn valid_input() -> f32 {
+ return loss_of(0.8, 0.3, 0.9, false);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test sigmoid_basic_case
+ given input = 0.0
+ when result = sigmoid(input)
+ then @abs(result - 0.5) < 1e-6
+
+ test sigmoid_positive_input
+ given input = 1.0
+ when result = sigmoid(input)
+ then result > 0.5 and result < 1.0 and @abs(result - 0.7310586) < 0.000001
+
+ test sigmoid_negative_input
+ given input = -1.0
+ when result = sigmoid(input)
+ then result > 0.0 and result < 0.5 and @abs(result + sigmoid(1.0) - 1.0) < 0.000001
+
+ // -(ln 0.8 + ln 0.7 + ln 0.9) / 3 = 0.2283930
+ test forward_basic_case
+ given result = loss_of(0.8, 0.3, 0.9, false)
+ then result > 0.0 and @abs(result - 0.2283930) < 0.000001
+
+ // From logits (2, -1, 3) it is the same loss as from their sigmoids
+ test forward_from_logits_case
+ given result = loss_of(2.0, -1.0, 3.0, true)
+ when probs = loss_of(sigmoid(2.0), sigmoid(-1.0), sigmoid(3.0), false)
+ then result > 0.0 and @abs(result - probs) < 0.000001
+
+ // (-ln 0.8 - ln 0.7 - ln 0.9 - ln 0.9) / 4 = 0.1976349
+ test forward_batch_basic_case
+ given result = batch_loss()
+ then result > 0.0 and @abs(result - 0.1976349) < 0.000001
+
+ // t = 1 pulls p up (negative gradient), t = 0 pushes it down (positive)
+ test gradient_basic_case
+ given g0 = grad_of(0.8, 0.3, 0.9, false, 0)
+ when g1 = grad_of(0.8, 0.3, 0.9, false, 1)
+ then g0 < 0.0 and g1 > 0.0 and grad_of(0.8, 0.3, 0.9, false, 2) < 0.0 and @abs(g0 + 1.25) < 0.000001 and @abs(g1 - 1.0 / 0.7) < 0.000001
+
+ // From logits: sigmoid(z) - t, and it matches a finite difference of the loss
+ test gradient_from_logits_case
+ given g1 = grad_of(2.0, -1.0, 3.0, true, 1)
+ when slope = 3.0 * (loss_of(2.0, -1.0 + 0.001, 3.0, true) - loss_of(2.0, -1.0 - 0.001, 3.0, true)) / 0.002
+ then @abs(g1 - sigmoid(-1.0)) < 0.000001 and @abs(g1 - slope) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // targets[i] in {0, 1}: a perfect prediction costs (almost) nothing
+ invariant binary_ce_constraint_0
+ assert loss_of(1.0, 0.0, 1.0, false) < 0.000001
+
+ // 0 < epsilon << 1
+ invariant binary_ce_constraint_1
+ assert DEFAULT_EPSILON > 0.0 and DEFAULT_EPSILON < 0.001
+
+ // name: binary_ce -- positive for an imperfect prediction
+ invariant binary_ce_constraint_2
+ assert valid_input() > 0.0
+
+ // name: gradient -- zero where the logit already gives the target
+ invariant binary_ce_constraint_3
+ assert @abs(grad_of(30.0, -30.0, 30.0, true, 0)) < 0.000001
+
+ // "Bishop (2006) - Pattern Recognition and Machine Learning": the loss falls as p nears t
+ invariant binary_ce_constraint_4
+ assert loss_of(0.9, 0.1, 0.9, false) < loss_of(0.6, 0.4, 0.6, false)
+
+ // "Goodfellow et al. (2016) - Deep Learning": clipping keeps a certain wrong answer finite
+ invariant binary_ce_constraint_5
+ assert loss_of(0.0, 1.0, 0.0, false) < 20.0
+
Why switching units off while training helps, and why the survivors must be scaled.
+
While training, dropout switches off a share p of the units at random, so the rest cannot lean on any one of them. The units that survive must then be scaled, or the next layer would see the signal at another size in training than at inference. Inverted dropout scales them by 1 / (1 - p) while training: with p = 0.5, input 1, 2, 3, 4 becomes 2, 0, 6, 0, and inference is a plain pass. Classic dropout keeps 1, 0, 3, 0 and scales by 0.5 at inference instead. The spec dropout_layer.t27 uses a fixed mask, keep, drop, keep, drop, so the numbers are the same every time.
+
Try it
Compare the outputs of the four modes; then read the gradient of a dropped unit in training.
Switch between training and inference, inverted and classic dropout: units 1 and 3 are dropped, the survivors are scaled so that the layer's output keeps its expected size.
+
specs/ml/layers/dropout_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Prevent co-adaptation of neurons | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the tests called `allocf32` and `allocbool`, which no spec declares, and
+// asserted `result != undefined`. Inference also scaled by (1 - p) even for
+// inverted dropout, which already scaled by 1 / (1 - p) in training, so the
+// expected activation shrank by (1 - p) between training and inference. Inference
+// is now the identity when scale_during_training is set.
+
+module Dropout;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_DROP_RATE : f32 = 0.5;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const DropoutConfig = struct {
+ p : f32,
+ inplace : bool,
+ scale_during_training : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(input: []const f32, output: []f32, mask: []bool, training: bool, config: DropoutConfig) → void
+ fn forward(input: []const f32, output: []f32, mask: []bool, training: bool, config: DropoutConfig) -> void {
+ const n = input.len;
+ var i : usize = 0;
+ while (i < n) {
+ if (training) {
+ if (mask[i]) {
+ // Neuron is kept, scale during training if configured
+ if (config.scale_during_training) {
+ output[i] = input[i] * (1.0 / (1.0 - config.p));
+ } else {
+ output[i] = input[i];
+ }
+ } else {
+ // Neuron is dropped
+ output[i] = 0.0;
+ }
+ } else {
+ if (config.scale_during_training) {
+ // Inverted dropout already scaled in training: inference is the identity
+ output[i] = input[i];
+ } else {
+ // Classic dropout: scale by the keep probability at inference
+ output[i] = input[i] * (1.0 - config.p);
+ }
+ }
+ i = i + 1;
+ }
+ }
+
+ // backward(grad_output: []const f32, mask: []const bool, grad_input: []f32, config: DropoutConfig) → void
+ fn backward(grad_output: []const f32, mask: []const bool, grad_input: []f32, config: DropoutConfig) -> void {
+ const n = grad_output.len;
+ var i : usize = 0;
+ while (i < n) {
+ if (mask[i]) {
+ // Gradient flows through kept neurons
+ if (config.scale_during_training) {
+ grad_input[i] = grad_output[i] * (1.0 / (1.0 - config.p));
+ } else {
+ grad_input[i] = grad_output[i];
+ }
+ } else {
+ // Gradient is zero for dropped neurons
+ grad_input[i] = 0.0;
+ }
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ fn cfg(scaled: bool) -> DropoutConfig {
+ return DropoutConfig{ .p = DEFAULT_DROP_RATE, .inplace = false, .scale_during_training = scaled };
+ }
+
+ // forward on input (1, 2, 3, 4) with mask (keep, drop, keep, drop)
+ fn forward_at(training: bool, scaled: bool, index: usize) -> f32 {
+ const input : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [4]f32 = [9.0] ** 4;
+ var mask : [4]bool = [true, false, true, false];
+ forward(&input, &output, &mask, training, cfg(scaled));
+ return output[index];
+ }
+
+ fn backward_at(scaled: bool, index: usize) -> f32 {
+ const grad_output : [4]f32 = [1.0, 1.0, 1.0, 1.0];
+ const mask : [4]bool = [true, false, true, false];
+ var grad_input : [4]f32 = [9.0] ** 4;
+ backward(&grad_output, &mask, &grad_input, cfg(scaled));
+ return grad_input[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // p = 0.5, inverted: kept units are doubled, dropped ones are 0
+ test forward_basic_case
+ given kept = forward_at(true, true, 2)
+ when dropped = forward_at(true, true, 1)
+ then kept == 6.0 and dropped == 0.0 and forward_at(true, true, 0) == 2.0
+
+ // The expected activation is the same in training and inference
+ test forward_inference_keeps_the_expectation
+ given inverted = forward_at(false, true, 2)
+ when classic = forward_at(false, false, 2)
+ then inverted == 3.0 and classic == 1.5 and forward_at(true, false, 2) == 3.0
+
+ // The gradient flows only through kept units, with the same scale
+ test backward_basic_case
+ given kept = backward_at(true, 0)
+ when dropped = backward_at(true, 3)
+ then kept == 2.0 and dropped == 0.0 and backward_at(false, 2) == 1.0
+
+ invariant dropout_rate_is_a_probability
+ assert DEFAULT_DROP_RATE >= 0.0 and DEFAULT_DROP_RATE < 1.0
+
+
+
+
+
+
diff --git a/apps/website/public/learn/elu/card.png b/apps/website/public/learn/elu/card.png
new file mode 100644
index 0000000000..a75e49ae4e
Binary files /dev/null and b/apps/website/public/learn/elu/card.png differ
diff --git a/apps/website/public/learn/elu/index.html b/apps/website/public/learn/elu/index.html
new file mode 100644
index 0000000000..600886bea4
--- /dev/null
+++ b/apps/website/public/learn/elu/index.html
@@ -0,0 +1,197 @@
+
+
+
+
+
+
+ELU: a floor, not a cliff: neural networks course, lesson 6 of 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiРусский
+
+
+
ELU: a floor, not a cliff
+
You will learn
How ELU keeps a soft negative side with a floor at -alpha, and why its slope never quite dies.
+
ELU keeps the positive side of ReLU and replaces the cliff on the negative side with alpha (e^x - 1), a curve that flattens toward -alpha. With alpha 1, x = -3 gives -0.9502, already near the floor of -1, and the slope there is 0.0498: small, but not 0, so the unit can still move. Because the negative outputs are not thrown away, a layer's outputs average closer to 0. The spec elu_activation.t27 computes the curve, the slope and the floor, and its own batch at -1, 0 and 2: -0.632, 0 and 2.
+
Try it
Set alpha to 0.5 and read the floor; then find how far left x must go before the slope is under 0.01.
Drag along ELU: x above zero, alpha (e^x - 1) below. The negative side never falls under -alpha, so outputs average closer to zero, and its slope never quite dies.
+
specs/ml/activation/elu_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Derivative | φ² + 1/φ² = 3 | TRINITY
+// #8980: `math::exp` named a module this spec never imports, forward_batch grew
+// an array with `+`, and every test called a two-argument function with one
+// argument and asserted `result != undefined`. Now each test checks a value.
+
+module Elu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_ALPHA : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const ELUConfig = struct {
+ alpha : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(x, alpha) = x for x > 0, alpha * (exp(x) - 1) otherwise
+ fn forward(x: f32, alpha: f32) -> f32 {
+ if (x > 0.0) {
+ return x;
+ }
+ return alpha * (exp(x) - 1.0);
+ }
+
+ // forward_batch: output[i] = forward(input[i], alpha)
+ fn forward_batch(input: []const f32, alpha: f32, output: []f32) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = forward(input[i], alpha);
+ i = i + 1;
+ }
+ }
+
+ // derivative(x, alpha) = 1 for x > 0, alpha * exp(x) otherwise
+ fn derivative(x: f32, alpha: f32) -> f32 {
+ if (x > 0.0) {
+ return 1.0;
+ }
+ return alpha * exp(x);
+ }
+
+ // The inputs the tests and invariants share: one negative, zero, one positive.
+ fn negative_input() -> f32 {
+ return -1.0;
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
+ }
+
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [-1.0, 0.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, DEFAULT_ALPHA, &output);
+ return output[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // ELU(-1) = exp(-1) - 1 = -0.6321206; ELU(2) = 2
+ test "forward_basic_case"
+ given input = negative_input()
+ when result = forward(input, DEFAULT_ALPHA)
+ then @abs(result - (-0.6321206)) < 0.000001 and forward(valid_input(), DEFAULT_ALPHA) == 2.0
+
+ // alpha scales only the negative side, and ELU(x) never goes below -alpha (in f32
+ // ELU(-20) rounds to exactly -alpha)
+ test "forward_alpha_scales_negative_side"
+ given input = negative_input()
+ when result = forward(input, 0.5)
+ then @abs(result - 0.5 * forward(input, 1.0)) < 0.000001 and forward(-20.0, 0.5) >= -0.5
+
+ test "forward_batch_basic_case"
+ given input = negative_input()
+ when result = batch_at(0)
+ then result == forward(input, DEFAULT_ALPHA) and batch_at(1) == 0.0 and batch_at(2) == 2.0
+
+ // ELU'(-1) = exp(-1) = 0.3678794; ELU'(2) = 1
+ test "derivative_basic_case"
+ given input = negative_input()
+ when result = derivative(input, DEFAULT_ALPHA)
+ then @abs(result - 0.3678794) < 0.000001 and derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+
+ test "derivative_matches_finite_difference"
+ given slope = (forward(-1.0 + 0.001, DEFAULT_ALPHA) - forward(-1.0 - 0.001, DEFAULT_ALPHA)) / 0.002
+ when result = derivative(-1.0, DEFAULT_ALPHA)
+ then @abs(result - slope) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ invariant elu_constraint_0
+ assert DEFAULT_ALPHA > 0.0
+
+ // name: elu_formula -- identity on the positive side
+ invariant elu_constraint_1
+ assert forward(valid_input(), DEFAULT_ALPHA) == valid_input()
+
+ // name: derivative -- 1 on the positive side
+ invariant elu_constraint_2
+ assert derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+
+ // "Clevert et al. (2016) - Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)":
+ // ELU is continuous at 0
+ invariant elu_constraint_3
+ assert forward(0.0, DEFAULT_ALPHA) == 0.0
+
How a token becomes a row of numbers, and why learning changes only the rows that were read.
+
A network cannot multiply a word, so each token is an index into a table with one row of numbers per token, and looking a token up copies its row. In the spec the table has 4 tokens of width 2 and row v holds (10 v, 10 v + 1), so token 2 reads (20, 21). Learning sends the gradient back only to the rows that were read, and a token read twice collects both: gradient rows (1, 2) and (3, 4) on token 2 add up to (4, 6). The spec embedding_layer.t27 computes the lookup and the gradient.
+
Try it
Read token 2 twice and then tokens 1 and 3; count the rows whose gradient is not 0 each time.
Pick two tokens: each one copies its row out of the table. Then the gradient flows back only into the rows that were read, and a token read twice collects both.
+
specs/ml/layers/embedding_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Embeddings are learned parameters | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward only asserted its arguments and wrote nothing, the tests called
+// undeclared `test_input_forward` / `test_input_init` and asserted
+// `result != undefined`, and the invariants asserted `true`. forward now looks each
+// token's row up in W (vocab_size x d_model, row-major), backward adds each
+// gradient row back into the row it came from, and init fills W deterministically.
+
+module Embedding;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_D_MODEL : u32 = 64;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const EmbeddingConfig = struct {
+ vocab_size : u32,
+ d_model : u32,
+ };
+
+ pub const EmbeddingWeights = struct {
+ W : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // embeddings[t * d + j] = W[token_ids[t] * d + j]: one row of W per token
+ fn forward(token_ids: []const u32, weights: EmbeddingWeights, embeddings: []f32, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ embeddings[t * d + j] = weights.W[row * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
+ }
+ }
+
+ // grad_W[token_ids[t]] += grad_embeddings[t]: a token seen twice collects both rows
+ fn backward(token_ids: []const u32, grad_embeddings: []const f32, grad_W: []f32, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ grad_W[row * d + j] = grad_W[row * d + j] + grad_embeddings[t * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
+ }
+ }
+
+ // Deterministic init in [-0.5, 0.5) / sqrt(d_model): a fixed hash of (row, column), so
+ // two runs start from the same weights
+ fn init(weights: EmbeddingWeights, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d)));
+ var k : usize = 0;
+ while (k < config.vocab_size * d) {
+ const h : u32 = @as(u32, @intCast((k * 2654435761) % 1000));
+ weights.W[k] = (@as(f32, @floatFromInt(h)) / 1000.0 - 0.5) * scale;
+ k = k + 1;
+ }
+ }
+
+ // vocab 4, d_model 2, W row v = (10 v, 10 v + 1)
+ fn cfg() -> EmbeddingConfig {
+ return EmbeddingConfig{ .vocab_size = 4, .d_model = 2 };
+ }
+
+ fn lookup_at(a: u32, b: u32, index: usize) -> f32 {
+ var w : [8]f32 = [0.0, 1.0, 10.0, 11.0, 20.0, 21.0, 30.0, 31.0];
+ const ids : [2]u32 = [a, b];
+ var out : [4]f32 = [-1.0] ** 4;
+ forward(&ids, EmbeddingWeights{ .W = &w }, &out, cfg());
+ return out[index];
+ }
+
+ // grad_W[index] after backward of ids (a, b) with gradient rows (1, 2) and (3, 4)
+ fn grad_at(a: u32, b: u32, index: usize) -> f32 {
+ const ids : [2]u32 = [a, b];
+ const grads : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_w : [8]f32 = [0.0] ** 8;
+ backward(&ids, &grads, &grad_w, cfg());
+ return grad_w[index];
+ }
+
+ fn init_at(index: usize) -> f32 {
+ var w : [8]f32 = [9.0] ** 8;
+ init(EmbeddingWeights{ .W = &w }, cfg());
+ return w[index];
+ }
+
+ fn valid_input() -> u32 {
+ return cfg().vocab_size * cfg().d_model;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // ids (2, 0): rows (20, 21) then (0, 1)
+ test forward_basic_case
+ given first = lookup_at(2, 0, 0)
+ when last = lookup_at(2, 0, 3)
+ then first == 20.0 and lookup_at(2, 0, 1) == 21.0 and lookup_at(2, 0, 2) == 0.0 and last == 1.0
+
+ // Every weight is written, inside [-0.5, 0.5) / sqrt(2), and init is reproducible
+ test init_basic_case
+ given first = init_at(0)
+ when last = init_at(7)
+ then first != 9.0 and last != 9.0 and @abs(first) <= 0.36 and @abs(last) <= 0.36 and init_at(5) == init_at(5) and init_at(1) != init_at(2)
+
+ // A token seen twice collects both gradient rows; an unseen row stays 0
+ test backward_accumulates_repeated_tokens
+ given row1 = grad_at(1, 1, 2)
+ when unseen = grad_at(1, 1, 0)
+ then row1 == 4.0 and grad_at(1, 1, 3) == 6.0 and unseen == 0.0 and grad_at(1, 3, 6) == 3.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // vocab_size > 0
+ invariant embedding_constraint_0
+ assert cfg().vocab_size > 0
+
+ // d_model > 0
+ invariant embedding_constraint_1
+ assert cfg().d_model > 0 and DEFAULT_D_MODEL > 0
+
+ // W.size() == vocab_size * d_model
+ invariant embedding_constraint_2
+ assert valid_input() == 8
+
+ // token_ids must be in [0, vocab_size): the last id reads the last row
+ invariant embedding_constraint_3
+ assert lookup_at(3, 3, 1) == 31.0
+
+ // "Embedding dimension affects model capacity": each token gets d_model numbers
+ invariant embedding_constraint_4
+ assert lookup_at(1, 2, 1) == 11.0 and lookup_at(1, 2, 2) == 20.0
+
+ // "Larger vocab requires larger d_model": the init scale shrinks as 1 / sqrt(d_model)
+ invariant embedding_constraint_5
+ assert 1.0 / sqrt(@as(f32, 64.0)) < 1.0 / sqrt(@as(f32, 2.0))
+
+ // "Pre-trained embeddings transfer well": forward is a pure lookup, so loaded rows come out unchanged
+ invariant embedding_constraint_6
+ assert lookup_at(0, 1, 0) == 0.0 and lookup_at(0, 1, 2) == 10.0
+
What GELU computes, why it dips below zero, and how close its tanh shortcut comes.
+
GELU weighs x by the chance that a standard normal draw is below it: GELU(x) = x Phi(x). A large positive input passes, a large negative one vanishes, and in between the curve dips to about -0.17 near x = -0.75 before it rises. BERT and GPT use it in their feed-forward layers. The exact form needs erf, which the spec computes with the Abramowitz-Stegun fit, good to 1.5e-7, and it also computes the tanh shortcut: at -0.75 they give -0.16997 and -0.17004. The spec gelu_activation.t27 computes both curves and their slopes.
+
Try it
Find the lowest point of the curve by its slope; then compare the exact and the tanh form at -3.
Drag along GELU: x weighed by the chance a normal draw is below it. It dips to about -0.17 before it rises, and the tanh shortcut stays within a few thousandths of the exact form.
+
specs/ml/activation/gelu_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Based on cumulative distribution function | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: Zig has neither `@tanh` nor `@erf`, so this never compiled. erf is now
+// a function of this spec (Abramowitz-Stegun 7.1.26, |error| < 1.5e-7), and the
+// exact derivative divides the normal density by sqrt(2 pi), not sqrt(2) (the
+// slip was invisible at x = 0, the only point the tests checked).
+
+module Gelu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const SQRT_2_OVER_PI : f32 = 0.7978845608;
+ const SQRT_PI_OVER_2 : f32 = 1.2533141373;
+ const SQRT_2 : f32 = 1.4142135624;
+ const SQRT_2_PI : f32 = 2.5066282746;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const GELUConfig = struct {
+ approximate : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // erf(x), Abramowitz-Stegun 7.1.26: max absolute error 1.5e-7
+ fn erf(x: f32) -> f32 {
+ if (x < 0.0) {
+ return -erf(-x);
+ }
+ const t = 1.0 / (1.0 + 0.3275911 * x);
+ const poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))));
+ return 1.0 - poly * exp(-x * x);
+ }
+
+ // One element. approximate: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3)));
+ // exact: 0.5 x (1 + erf(x / sqrt(2))) = x Phi(x)
+ fn gelu_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ return 0.5 * x * (1.0 + tanh(inner));
+ }
+ return 0.5 * x * (1.0 + erf(x / SQRT_2));
+ }
+
+ // d/dx of gelu_one
+ fn gelu_grad_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ const tanh_val = tanh(inner);
+ const sech_sq = 1.0 - tanh_val * tanh_val;
+ return 0.5 * (1.0 + tanh_val) + 0.5 * x * sech_sq * SQRT_2_OVER_PI * (1.0 + 3.0 * 0.044715 * x * x);
+ }
+ // Phi(x) + x phi(x), phi(x) = exp(-x^2 / 2) / sqrt(2 pi)
+ const cdf = 0.5 * (1.0 + erf(x / SQRT_2));
+ const pdf = exp(-0.5 * x * x) / SQRT_2_PI;
+ return cdf + x * pdf;
+ }
+
+ fn forward(input: []const f32, output: []f32, config: GELUConfig) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = gelu_one(input[i], config.approximate);
+ i = i + 1;
+ }
+ }
+
+ fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: GELUConfig) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ grad_input[i] = grad_output[i] * gelu_grad_one(input[i], config.approximate);
+ i = i + 1;
+ }
+ }
+
+ // forward / backward over a one-element slice, for the tests
+ fn forward_at(x: f32, approximate: bool) -> f32 {
+ const input : [1]f32 = [x];
+ var output : [1]f32 = [0.0];
+ forward(&input, &output, GELUConfig{ .approximate = approximate });
+ return output[0];
+ }
+
+ fn backward_at(x: f32, approximate: bool) -> f32 {
+ const grad_output : [1]f32 = [1.0];
+ const input : [1]f32 = [x];
+ var grad_input : [1]f32 = [0.0];
+ backward(&grad_output, &input, &grad_input, GELUConfig{ .approximate = approximate });
+ return grad_input[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // The product goes through a function so the check runs, not only folds at compile time.
+ fn product_of(a: f32, b: f32) -> f32 {
+ return a * b;
+ }
+
+ fn config_of(approximate: bool) -> GELUConfig {
+ return GELUConfig{ .approximate = approximate };
+ }
+
+ test gelu_sqrt_constants_are_reciprocals
+ given product = product_of(SQRT_2_OVER_PI, SQRT_PI_OVER_2)
+ then @abs(product - 1.0) < 0.0000005
+
+ test gelu_sqrt_2_over_pi_squares_to_2_over_pi
+ given squared = product_of(SQRT_2_OVER_PI, SQRT_2_OVER_PI)
+ then @abs(squared - 0.63661977) < 0.0000003
+
+ test gelu_config_records_the_approximation_choice
+ given exact = config_of(false)
+ when approx = config_of(true)
+ then !exact.approximate and approx.approximate
+
+ // erf(0) = 0, erf(1) = 0.8427008, erf(-1) = -erf(1)
+ test gelu_erf_known_values
+ given at_one = erf(1.0)
+ when at_zero = erf(0.0)
+ then @abs(at_zero) < 0.0000002 and @abs(at_one - 0.8427008) < 0.0000003 and erf(-1.0) == -at_one
+
+ test gelu_forward_zero_input
+ given exact = forward_at(0.0, false)
+ when approx = forward_at(0.0, true)
+ then exact == 0.0 and approx == 0.0
+
+ test gelu_backward_zero_input_derivative
+ given exact = backward_at(0.0, false)
+ when approx = backward_at(0.0, true)
+ then @abs(exact - 0.5) < 0.0000002 and approx == 0.5
+
+ test gelu_forward_monotonic_positive_tail
+ given small = forward_at(1.0, true)
+ when large = forward_at(2.0, true)
+ then small < large
+
+ // With the cubic term, approximate GELU(1) = 0.8411920; exact GELU(1) = Phi(1) = 0.8413447.
+ // Without it, 0.5 (1 + tanh(0.7978846)) = 0.8296438 -- so this pins the term.
+ test gelu_tanh_cubic_coefficient
+ given approx = forward_at(1.0, true)
+ when exact = forward_at(1.0, false)
+ then @abs(approx - 0.8411920) < 0.000001 and @abs(exact - 0.8413447) < 0.000001
+
+ test gelu_backward_exact_derivative_at_zero
+ given result = backward_at(0.0, false)
+ then @abs(result - 0.5) < 0.0000002
+
+ test gelu_forward_approx_derivative_at_zero
+ given result = backward_at(0.0, true)
+ then result == 0.5
+
+ // The derivative at x = 1 is where the old sqrt(2) slip showed: 1.0833155 exact
+ test gelu_backward_matches_finite_difference
+ given exact_slope = (forward_at(1.001, false) - forward_at(0.999, false)) / 0.002
+ when approx_slope = (forward_at(1.001, true) - forward_at(0.999, true)) / 0.002
+ then @abs(backward_at(1.0, false) - exact_slope) < 0.002 and @abs(backward_at(1.0, true) - approx_slope) < 0.002 and @abs(backward_at(1.0, false) - 1.0833155) < 0.000002
+
How gradient descent walks down a loss, what momentum adds, and why too large a rate never settles.
+
Training is walking downhill on the loss. Gradient descent moves a weight by -lr times its gradient each step; in the spec the loss is (x - 3)^2 from x = 0, and lr 0.1 reaches 2.99996 in 50 steps. Momentum adds a share of the last move, which speeds up a long slope and overshoots a sharp bottom: at 0.9 the weight shoots past 3 to about 5.13 and rings down to 3.09 after 50 steps. Nesterov takes the gradient one step ahead and swings less, to about 4.25. Too large a rate never settles: at lr 1 the weight bounces between 0 and 6 for good. The spec sgd.t27 computes every step.
+
Try it
Find the largest learning rate that still settles without momentum; then add momentum 0.9 and count the swings.
Set the learning rate and momentum and watch a weight walk from 0 to the bottom of (x - 3)^2. Plain steps creep, momentum overshoots and rings, and a rate of 1 bounces forever.
+
specs/ml/optimizer/sgd.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Nesterov accelerated gradient (NAG) | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: init called `new f32[n]`; update read undeclared locals; dampening and
+// nesterov were ignored; and weight decay was ADDED to the parameter
+// (`- v + wd * p`), which grows the weights it should shrink. update is now
+// PyTorch's SGD: g = grad + wd p; v = mu v + (1 - dampening) g;
+// step = g + mu v (Nesterov) or v; p = p - lr step.
+
+module Sgd;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_LR : f32 = 0.01;
+ const DEFAULT_MOMENTUM : f32 = 0.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SGDConfig = struct {
+ learning_rate : f32,
+ momentum : f32,
+ weight_decay : f32,
+ dampening : f32,
+ nesterov : bool,
+ };
+
+ pub const SGDState = struct {
+ velocity : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // init: the first param_count velocities start at zero (the caller owns the buffer)
+ fn init(state: SGDState, param_count: u32) -> void {
+ var i : usize = 0;
+ while (i < param_count) {
+ state.velocity[i] = 0.0;
+ i = i + 1;
+ }
+ }
+
+ fn update(params: []f32, grads: []const f32, state: SGDState, config: SGDConfig) -> void {
+ var i : usize = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ var step = g;
+ if (config.momentum > 0.0) {
+ state.velocity[i] = config.momentum * state.velocity[i] + (1.0 - config.dampening) * g;
+ if (config.nesterov) {
+ step = g + config.momentum * state.velocity[i];
+ } else {
+ step = state.velocity[i];
+ }
+ }
+ params[i] = params[i] - config.learning_rate * step;
+ i = i + 1;
+ }
+ }
+
+ fn cfg(lr: f32, momentum: f32, wd: f32, nesterov: bool) -> SGDConfig {
+ return SGDConfig{ .learning_rate = lr, .momentum = momentum, .weight_decay = wd, .dampening = 0.0, .nesterov = nesterov };
+ }
+
+ // Minimise f(x) = (x - 3)^2 (gradient 2 (x - 3)) from x = 0 for `steps` steps
+ fn descend(lr: f32, momentum: f32, nesterov: bool, steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var velocity : [1]f32 = [5.0];
+ const state = SGDState{ .velocity = &velocity };
+ init(state, 1);
+ var n : u32 = 0;
+ while (n < steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ update(¶ms, &grads, state, cfg(lr, momentum, 0.0, nesterov));
+ n = n + 1;
+ }
+ return params[0];
+ }
+
+ fn init_zeroes(index: usize) -> f32 {
+ var velocity : [3]f32 = [7.0, 7.0, 7.0];
+ init(SGDState{ .velocity = &velocity }, 3);
+ return velocity[index];
+ }
+
+ // One step on params (1, 2), grads (0.1, 0.2), momentum 0.9, weight decay 0.001
+ fn one_step(index: usize) -> f32 {
+ var params : [2]f32 = [1.0, 2.0];
+ const grads : [2]f32 = [0.1, 0.2];
+ var velocity : [2]f32 = [0.0, 0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.01, 0.9, 0.001, false));
+ return params[index];
+ }
+
+ // A step with zero gradient and weight decay shrinks the weight toward 0
+ fn decay_step() -> f32 {
+ var params : [1]f32 = [2.0];
+ const grads : [1]f32 = [0.0];
+ var velocity : [1]f32 = [0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.1, 0.0, 0.5, false));
+ return params[0];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_LR;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test "init_basic_case"
+ given first = init_zeroes(0)
+ when last = init_zeroes(2)
+ then first == 0.0 and last == 0.0
+
+ // v = g + wd p (momentum starts at 0): p0 = 1 - 0.01 (0.1 + 0.001) = 0.99899
+ test "update_basic_case"
+ given p0 = one_step(0)
+ when p1 = one_step(1)
+ then p0 != 1.0 and p1 != 2.0 and @abs(p0 - 0.99899) < 0.000001 and @abs(p1 - (2.0 - 0.01 * 0.202)) < 0.000001
+
+ // Weight decay pulls the weight toward 0: 2 - 0.1 * 0.5 * 2 = 1.9
+ test "update_weight_decay_shrinks"
+ given result = decay_step()
+ then @abs(result - 1.9) < 0.000001
+
+ // Plain, momentum and Nesterov SGD all reach the minimum at 3
+ test "update_descends_a_quadratic"
+ given plain = descend(0.1, 0.0, false, 100)
+ when heavy = descend(0.05, 0.9, false, 300)
+ then @abs(plain - 3.0) < 0.0001 and @abs(heavy - 3.0) < 0.0001 and @abs(descend(0.05, 0.9, true, 300) - 3.0) < 0.0001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // learning_rate > 0
+ invariant sgd_constraint_0
+ assert valid_input() > 0.0
+
+ // 0 <= momentum < 1
+ invariant sgd_constraint_1
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+
+ // weight_decay >= 0: a non-negative decay never grows a weight with zero gradient
+ invariant sgd_constraint_2
+ assert decay_step() < 2.0
+
+ // params.size() == grads.size(): every parameter moves
+ invariant sgd_constraint_3
+ assert one_step(0) != 1.0 and one_step(1) != 2.0
+
+ // If momentum > 0: state.velocity.size() == params.size() -- init zeroes all of it
+ invariant sgd_constraint_4
+ assert init_zeroes(1) == 0.0
+
Before the FPGA course: what a t27 spec is, how the compiler reads it, and what it gives back. Every lesson opens one table or diagram drawn from its own spec and one spec you can run in your browser.
Course 0, 27 lessons, one infographic and one t27 spec each: a module, constants and types, functions and tests, then the compiler's stages and the 7 languages it writes. It comes before the FPGA course.
How LAMB scales each layer's step by its own weight norm, so one learning rate suits layers of every size.
+
Training with very large batches needs a large learning rate, and one rate can be too large for a small layer and too small for a big one. LAMB takes an Adam-style update for each layer and rescales it by the trust ratio, the layer's weight norm, capped at 10, over the update's norm, so every layer moves by the same fraction of itself. In the spec, the weights (1, 2) times a scale take a step whose norm is 0.001 times theirs: 0.00447 at scale 2, and once the weight norm passes 10, near scale 4.47, the step stays at 0.01. The spec lamb.t27 computes the update for one layer and for two at once.
+
Try it
Read the step at scales 1 and 2 and compare; then drag past the bend and explain why the line goes flat.
Scale a layer's weights and watch its LAMB step: it grows with the weight norm, so every layer moves by the same fraction of itself, until the norm passes 10 and the step stops growing.
+
specs/ml/optimizer/lamb.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// LAMB parameter update with trust ratio clipping | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: built with `append` on `[]` and `range` / `len` / `pow`, so it never
+// compiled. The math was also not LAMB: v_hat used beta1's bias correction, the
+// "trust ratio" was ||g|| / ||v_hat|| and was computed, then never applied to the
+// step, and m and v were never written back. It is now You et al. (2020):
+// r = m_hat / (sqrt(v_hat) + eps) + wd * w, trust = min(||w||, clip) / ||r||,
+// w = w - lr * trust * r,
+// per layer, over buffers the caller owns.
+
+module Lamb;
+ use base::types;
+ use math::constants;
+ use ml::optimizer::adam;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_LR : f32 = 0.001;
+ const DEFAULT_BETA1 : f32 = 0.9;
+ const DEFAULT_BETA2 : f32 = 0.999;
+ const DEFAULT_EPSILON : f32 = 1e-6;
+ const DEFAULT_CLIP_THRESHOLD : f32 = 10.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const LAMBConfig = struct {
+ learning_rate : f32,
+ beta1 : f32,
+ beta2 : f32,
+ epsilon : f32,
+ weight_decay : f32,
+ clip_threshold : f32,
+ };
+
+ pub const LAMBState = struct {
+ m : []f32,
+ v : []f32,
+ step : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // m and v are the first num_params entries of the given buffers, zeroed; step 0
+ fn init_state(m: []f32, v: []f32, num_params: u32) -> LAMBState {
+ var i : usize = 0;
+ while (i < num_params) {
+ m[i] = 0.0;
+ v[i] = 0.0;
+ i = i + 1;
+ }
+ return LAMBState{ .m = m[0..num_params], .v = v[0..num_params], .step = 0 };
+ }
+
+ // powi (b^t for a whole t) comes from specs/ml/optimizer/adam.t27 through `use`
+
+ // Writes the moments for step (state.step + 1) into state.m / state.v and the update to
+ // ADD to params into `update`; returns update[0..n]
+ fn compute_layer_update(params: []const f32, grads: []const f32, state: LAMBState, config: LAMBConfig, update: []f32) -> []f32 {
+ const n = params.len;
+ const t = state.step + 1;
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ var w_norm : f32 = 0.0;
+ var r_norm : f32 = 0.0;
+ var i : usize = 0;
+ while (i < n) {
+ state.m[i] = config.beta1 * state.m[i] + (1.0 - config.beta1) * grads[i];
+ state.v[i] = config.beta2 * state.v[i] + (1.0 - config.beta2) * grads[i] * grads[i];
+ const r = (state.m[i] / c1) / (sqrt(state.v[i] / c2) + config.epsilon) + config.weight_decay * params[i];
+ update[i] = r;
+ w_norm = w_norm + params[i] * params[i];
+ r_norm = r_norm + r * r;
+ i = i + 1;
+ }
+ w_norm = sqrt(w_norm);
+ r_norm = sqrt(r_norm);
+ var trust : f32 = 1.0;
+ if (w_norm > 0.0 and r_norm > 0.0) {
+ trust = @min(w_norm, config.clip_threshold) / r_norm;
+ }
+ i = 0;
+ while (i < n) {
+ update[i] = -config.learning_rate * trust * update[i];
+ i = i + 1;
+ }
+ return update[0..n];
+ }
+
+ // One LAMB step over several layers packed back to back in `params` (layer k has
+ // sizes[k] entries), each with its own state; params change in place, each step + 1.
+ fn forward(params: []f32, grads: []const f32, sizes: []const usize, states: []LAMBState, config: LAMBConfig, scratch: []f32) -> void {
+ var start : usize = 0;
+ var k : usize = 0;
+ while (k < sizes.len) {
+ const end = start + sizes[k];
+ const upd = compute_layer_update(params[start..end], grads[start..end], states[k], config, scratch);
+ var j : usize = 0;
+ while (j < sizes[k]) {
+ params[start + j] = params[start + j] + upd[j];
+ j = j + 1;
+ }
+ states[k].step = states[k].step + 1;
+ start = end;
+ k = k + 1;
+ }
+ }
+
+ fn cfg(lr: f32) -> LAMBConfig {
+ return LAMBConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.01, .clip_threshold = DEFAULT_CLIP_THRESHOLD };
+ }
+
+ // which: 0 = m.len, 1 = v.len, 2 = step, 3 = m[0], 4 = v[99]
+ fn init_probe(which: u8) -> f32 {
+ var m : [100]f32 = [9.0] ** 100;
+ var v : [100]f32 = [9.0] ** 100;
+ const st = init_state(&m, &v, 100);
+ if (which == 0) {
+ return @as(f32, @floatFromInt(st.m.len));
+ }
+ if (which == 1) {
+ return @as(f32, @floatFromInt(st.v.len));
+ }
+ if (which == 2) {
+ return @as(f32, @floatFromInt(st.step));
+ }
+ if (which == 3) {
+ return st.m[0];
+ }
+ return st.v[99];
+ }
+
+ // update[index] of the first step on params (1, 2, 3), grads (0.1, 0.2, 0.3); index 3 = its length
+ fn first_update(index: usize) -> f32 {
+ const params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [0.1, 0.2, 0.3];
+ var m : [3]f32 = undefined;
+ var v : [3]f32 = undefined;
+ var update : [3]f32 = [9.0] ** 3;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 3), cfg(0.001), &update);
+ if (index == 3) {
+ return @as(f32, @floatFromInt(u.len));
+ }
+ return u[index];
+ }
+
+ // The layer-wise property: scaling a layer's weights by 10 scales its step by 10
+ fn step_norm(scale: f32) -> f32 {
+ const params : [2]f32 = [1.0 * scale, 2.0 * scale];
+ const grads : [2]f32 = [0.1, 0.2];
+ var m : [2]f32 = undefined;
+ var v : [2]f32 = undefined;
+ var update : [2]f32 = undefined;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 2), cfg(0.001), &update);
+ return sqrt(u[0] * u[0] + u[1] * u[1]);
+ }
+
+ // Two layers (1, 2) and (3, 4), one step; which: 0..3 = new params, 4 = step of layer 1
+ fn two_layers(which: usize) -> f32 {
+ var params : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const grads : [4]f32 = [0.1, 0.2, 0.3, 0.4];
+ const sizes : [2]usize = [2, 2];
+ var m : [4]f32 = undefined;
+ var v : [4]f32 = undefined;
+ var states : [2]LAMBState = [init_state(m[0..2], v[0..2], 2), init_state(m[2..4], v[2..4], 2)];
+ var scratch : [2]f32 = undefined;
+ forward(¶ms, &grads, &sizes, &states, cfg(0.001), &scratch);
+ if (which == 4) {
+ return @as(f32, @floatFromInt(states[1].step));
+ }
+ return params[which];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_BETA1;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test init_state_basic_case
+ given num_params = 100.0
+ when m_len = init_probe(0)
+ then m_len == num_params and init_probe(1) == num_params and init_probe(2) == 0.0 and init_probe(3) == 0.0 and init_probe(4) == 0.0
+
+ // Every update is negative (descent) and the step's norm is lr * min(||w||, 10) = 0.001 ||w||
+ test compute_layer_update_basic_case
+ given u0 = first_update(0)
+ when n = first_update(3)
+ then n == 3.0 and u0 < 0.0 and first_update(1) < 0.0 and first_update(2) < 0.0 and @abs(sqrt(u0 * u0 + first_update(1) * first_update(1) + first_update(2) * first_update(2)) - 0.001 * sqrt(@as(f32, 14.0))) < 0.0000001
+
+ // Each layer moves, and each layer's state advances
+ test forward_basic_case
+ given p0 = two_layers(0)
+ when p3 = two_layers(3)
+ then p0 != 1.0 and two_layers(1) != 2.0 and two_layers(2) != 3.0 and p3 != 4.0 and p0 < 1.0 and two_layers(4) == 1.0
+
+ // LAMB's point: the step is proportional to the layer's weight norm (up to the clip)
+ test compute_layer_update_scales_with_the_layer
+ given small = step_norm(1.0)
+ when large = step_norm(3.0)
+ then @abs(large / small - 3.0) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // 0 < beta1 < 1
+ invariant lamb_constraint_0
+ assert valid_input() > 0.0 and valid_input() < 1.0
+
+ // 0 < beta2 < 1
+ invariant lamb_constraint_1
+ assert DEFAULT_BETA2 > 0.0 and DEFAULT_BETA2 < 1.0
+
+ // epsilon > 0
+ invariant lamb_constraint_2
+ assert DEFAULT_EPSILON > 0.0
+
+ // clip_threshold > 0
+ invariant lamb_constraint_3
+ assert DEFAULT_CLIP_THRESHOLD > 0.0
+
+ // name: trust_ratio -- capped by clip_threshold: a layer of norm 50 steps as one of norm 10
+ invariant lamb_constraint_4
+ assert DEFAULT_CLIP_THRESHOLD == 10.0
+
+ // name: lamb_update -- bias correction makes the first m_hat equal g
+ invariant lamb_constraint_5
+ assert powi(DEFAULT_BETA1, 1) == DEFAULT_BETA1 and DEFAULT_LR > 0.0
+
+ // "You et al. (2020) - Large Batch Optimization for Deep Learning: Training BERT in 76 minutes"
+ invariant lamb_constraint_6
+ assert powi(0.5, 3) == 0.125
+
+ // "Used in BERT, T5, and other large-scale Transformer training"
+ invariant lamb_constraint_7
+ assert DEFAULT_BETA2 > DEFAULT_BETA1
+
Why one neuron cannot learn XOR, and how a hidden layer trained by backpropagation does.
+
XOR is 1 when exactly one of two inputs is 1. No straight line separates the cases that give 1 from those that give 0, so a single neuron cannot learn it. A hidden layer bends the space: the spec trains a 2-8-1 network of ReLUs by backpropagation, lr 0.05, from fixed initial weights. Its loss falls from 0.8896 in epoch 1 to 1.8e-13 after 2000 epochs, and its four answers come out within 4e-7 of 0, 1, 1 and 0. The widget runs that training in your browser, from scratch for every point of the curve. The spec mlp.t27 holds the network and the test that it learns.
+
Try it
Drag the epochs back to 100 and read the four answers; then find the epoch where the curve stops falling.
A 2-8-1 network trained by backpropagation in your browser: drag the epochs from 0 to 2000 and watch its four answers move to 0, 1, 1, 0 while the loss falls from 0.89 by twelve orders.
+
specs/ml/pathway/mlp.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// ReLU activation for hidden layers | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward computed the hidden layer and wrote it into `output` (the
+// second layer never ran); init called an undeclared `alloc_f32`; the tests
+// called undeclared `create_test_*` helpers; and the invariants asserted `true`.
+// The MLP is now two layers, h = relu(W1 x + b1) and y = W2 h + b2, over buffers
+// the caller owns. train_step adds backpropagation with an SGD update, and the
+// tests train the network on XOR, which no single linear layer can fit.
+
+module Mlp;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const MNIST_INPUT_SIZE : u32 = 784;
+ const MNIST_OUTPUT_SIZE : u32 = 10;
+ const DEFAULT_HIDDEN_SIZE : u32 = 128;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const MLPConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ output_size : u32,
+ };
+
+ pub const MLPState = struct {
+ w1 : []f32,
+ b1 : []f32,
+ w2 : []f32,
+ b2 : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // One dense layer: out[o] = sum_i w[o * n_in + i] in[i] + b[o], relu when asked
+ fn layer(input: []const f32, w: []const f32, b: []const f32, n_in: u32, n_out: u32, relu: bool, out: []f32) -> void {
+ var o : usize = 0;
+ while (o < n_out) {
+ var sum : f32 = b[o];
+ var i : usize = 0;
+ while (i < n_in) {
+ sum = sum + w[o * n_in + i] * input[i];
+ i = i + 1;
+ }
+ if (relu and sum < 0.0) {
+ sum = 0.0;
+ }
+ out[o] = sum;
+ o = o + 1;
+ }
+ }
+
+ // h = relu(W1 x + b1) into `hidden`, y = W2 h + b2 into `output`
+ fn forward(input: []const f32, state: MLPState, hidden: []f32, output: []f32, config: MLPConfig) -> void {
+ layer(input, state.w1, state.b1, config.input_size, config.hidden_size, true, hidden);
+ layer(hidden, state.w2, state.b2, config.hidden_size, config.output_size, false, output);
+ }
+
+ // He-scaled deterministic init: w in [-1, 1) sqrt(2 / fan_in) from a fixed hash of the
+ // index (salt tells the layers apart); every bias 0
+ fn init_weights(w: []f32, fan_in: u32, salt: usize) -> void {
+ const scale : f32 = sqrt(2.0 / @as(f32, @floatFromInt(fan_in)));
+ var k : usize = 0;
+ while (k < w.len) {
+ const h : u32 = @as(u32, @intCast(((k + salt) * 2654435761) % 1000));
+ w[k] = (@as(f32, @floatFromInt(h)) / 500.0 - 1.0) * scale;
+ k = k + 1;
+ }
+ }
+
+ fn zero(v: []f32) -> void {
+ var k : usize = 0;
+ while (k < v.len) {
+ v[k] = 0.0;
+ k = k + 1;
+ }
+ }
+
+ fn init(state: MLPState, config: MLPConfig) -> void {
+ init_weights(state.w1, config.input_size, 1);
+ zero(state.b1);
+ init_weights(state.w2, config.hidden_size, 7);
+ zero(state.b2);
+ }
+
+ // One SGD step on the loss 0.5 * sum_o (y_o - t_o)^2; returns the loss before the step.
+ // d2_o = y_o - t_o (output error)
+ // d1_i = [z_i > 0] sum_o W2[o][i] d2_o (backpropagated through relu)
+ // W2 -= lr d2 h^T, b2 -= lr d2; W1 -= lr d1 x^T, b1 -= lr d1
+ fn train_step(input: []const f32, target: []const f32, state: MLPState, hidden: []f32, delta: []f32, output: []f32, lr: f32, config: MLPConfig) -> f32 {
+ forward(input, state, hidden, output, config);
+ var loss : f32 = 0.0;
+ var o : usize = 0;
+ while (o < config.output_size) {
+ const e = output[o] - target[o];
+ loss = loss + 0.5 * e * e;
+ o = o + 1;
+ }
+ var i : usize = 0;
+ while (i < config.hidden_size) {
+ var back : f32 = 0.0;
+ if (hidden[i] > 0.0) {
+ o = 0;
+ while (o < config.output_size) {
+ back = back + state.w2[o * config.hidden_size + i] * (output[o] - target[o]);
+ o = o + 1;
+ }
+ }
+ delta[i] = back;
+ i = i + 1;
+ }
+ o = 0;
+ while (o < config.output_size) {
+ const d2 = output[o] - target[o];
+ i = 0;
+ while (i < config.hidden_size) {
+ state.w2[o * config.hidden_size + i] = state.w2[o * config.hidden_size + i] - lr * d2 * hidden[i];
+ i = i + 1;
+ }
+ state.b2[o] = state.b2[o] - lr * d2;
+ o = o + 1;
+ }
+ i = 0;
+ while (i < config.hidden_size) {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ state.w1[i * config.input_size + j] = state.w1[i * config.input_size + j] - lr * delta[i] * input[j];
+ j = j + 1;
+ }
+ state.b1[i] = state.b1[i] - lr * delta[i];
+ i = i + 1;
+ }
+ return loss;
+ }
+
+ fn mnist() -> MLPConfig {
+ return MLPConfig{ .input_size = MNIST_INPUT_SIZE, .hidden_size = DEFAULT_HIDDEN_SIZE, .output_size = MNIST_OUTPUT_SIZE };
+ }
+
+ // which: 0 = output[index], 1 = min over every output, 2 = number of outputs written,
+ // on the MNIST shape with input pixel k = (k % 7) / 7
+ fn mnist_forward(which: u8, index: usize) -> f32 {
+ var w1 : [784 * 128]f32 = undefined;
+ var b1 : [128]f32 = undefined;
+ var w2 : [128 * 10]f32 = undefined;
+ var b2 : [10]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, mnist());
+ var input : [784]f32 = undefined;
+ var k : usize = 0;
+ while (k < 784) {
+ input[k] = @as(f32, @floatFromInt(k % 7)) / 7.0;
+ k = k + 1;
+ }
+ var hidden : [128]f32 = undefined;
+ var output : [11]f32 = [-99.0] ** 11;
+ forward(&input, state, &hidden, &output, mnist());
+ if (which == 0) {
+ return output[index];
+ }
+ var min_h : f32 = hidden[0];
+ var written : f32 = 0.0;
+ k = 0;
+ while (k < 128) {
+ min_h = @min(min_h, hidden[k]);
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 11) {
+ if (output[k] != -99.0) {
+ written = written + 1.0;
+ }
+ k = k + 1;
+ }
+ if (which == 1) {
+ return min_h;
+ }
+ return written;
+ }
+
+ // which: 0 = w1.len written by init, 1 = b1[0], 2 = b2[0], 3 = largest |w1|
+ fn mnist_init(which: u8) -> f32 {
+ var w1 : [784 * 128]f32 = [9.0] ** (784 * 128);
+ var b1 : [128]f32 = [9.0] ** 128;
+ var w2 : [128 * 10]f32 = [9.0] ** (128 * 10);
+ var b2 : [10]f32 = [9.0] ** 10;
+ init(MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, mnist());
+ var written : f32 = 0.0;
+ var largest : f32 = 0.0;
+ var k : usize = 0;
+ while (k < w1.len) {
+ if (w1[k] != 9.0) {
+ written = written + 1.0;
+ }
+ largest = @max(largest, @abs(w1[k]));
+ k = k + 1;
+ }
+ if (which == 0) {
+ return written;
+ }
+ if (which == 1) {
+ return b1[0];
+ }
+ if (which == 2) {
+ return b2[0];
+ }
+ return largest;
+ }
+
+ // Train a 2-8-1 network on XOR for `epochs` passes over the four cases.
+ // which: 0 = total loss of the first epoch, 1 = total loss of the last epoch,
+ // 2 + c = the trained network's output for case c
+ fn xor(epochs: u32, which: usize) -> f32 {
+ const cfg = MLPConfig{ .input_size = 2, .hidden_size = 8, .output_size = 1 };
+ var w1 : [16]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [8]f32 = undefined;
+ var b2 : [1]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, cfg);
+ const xs : [8]f32 = [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0];
+ const ts : [4]f32 = [0.0, 1.0, 1.0, 0.0];
+ var hidden : [8]f32 = undefined;
+ var delta : [8]f32 = undefined;
+ var output : [1]f32 = undefined;
+ var first : f32 = 0.0;
+ var last : f32 = 0.0;
+ var e : u32 = 0;
+ while (e < epochs) {
+ var total : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ total = total + train_step(xs[c * 2 .. c * 2 + 2], ts[c .. c + 1], state, &hidden, &delta, &output, 0.05, cfg);
+ c = c + 1;
+ }
+ if (e == 0) {
+ first = total;
+ }
+ last = total;
+ e = e + 1;
+ }
+ if (which == 0) {
+ return first;
+ }
+ if (which == 1) {
+ return last;
+ }
+ forward(xs[(which - 2) * 2 .. (which - 2) * 2 + 2], state, &hidden, &output, cfg);
+ return output[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // 784 -> 128 -> 10: exactly the 10 outputs are written, the 11th slot is untouched
+ test forward_output_size
+ given written = mnist_forward(2, 0)
+ when spare = mnist_forward(0, 10)
+ then written == 10.0 and spare == -99.0
+
+ // The hidden layer is relu, so no hidden unit is negative
+ test forward_relu_activation
+ given min_hidden = mnist_forward(1, 0)
+ then min_hidden >= 0.0
+
+ // init writes every one of the 784 * 128 first-layer weights, each within sqrt(2 / 784) = 0.05051
+ test init_allocation_sizes
+ given written = mnist_init(0)
+ when largest = mnist_init(3)
+ then written == 100352.0 and largest <= 0.0506 and largest > 0.0
+
+ test init_bias_initialization
+ given b1 = mnist_init(1)
+ when b2 = mnist_init(2)
+ then b1 == 0.0 and b2 == 0.0
+
+ // Backpropagation learns XOR: the loss falls by more than 100x, and each output is within 0.1
+ test train_step_learns_xor
+ given first = xor(2000, 0)
+ when last = xor(2000, 1)
+ then last < first / 100.0 and @abs(xor(2000, 2)) < 0.1 and @abs(xor(2000, 3) - 1.0) < 0.1 and @abs(xor(2000, 4) - 1.0) < 0.1 and @abs(xor(2000, 5)) < 0.1
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input_size == MNIST_INPUT_SIZE (784)
+ invariant mlp_constraint_0
+ assert mnist().input_size == 784
+
+ // output_size == MNIST_OUTPUT_SIZE (10)
+ invariant mlp_constraint_1
+ assert mnist().output_size == 10
+
+ // hidden_size > 0
+ invariant mlp_constraint_2
+ assert mnist().hidden_size > 0
+
+ // w1.size() == input_size * hidden_size
+ invariant mlp_constraint_3
+ assert mnist().input_size * mnist().hidden_size == 100352
+
+ // w2.size() == hidden_size * output_size
+ invariant mlp_constraint_4
+ assert mnist().hidden_size * mnist().output_size == 1280
+
+ // b1.size() == hidden_size
+ invariant mlp_constraint_5
+ assert DEFAULT_HIDDEN_SIZE == 128
+
+ // b2.size() == output_size
+ invariant mlp_constraint_6
+ assert MNIST_OUTPUT_SIZE == 10
+
How the forget, input and output gates of an LSTM decide what the cell keeps, writes and shows.
+
An LSTM cell keeps a separate memory c and three gates, each a sigmoid between 0 and 1: forget scales the old memory, input lets in new content g, and output decides how much of the memory to show. c' = f c + i g and h' = o tanh(c'). In the spec's fixture every weight is 0, so each gate is the sigmoid of its bias: with the forget gate at 0.9975 and the input gate at 0.0025, a memory of 1 comes out as 0.9975, and the output is 0.7587. Because c passes through with only a multiply by f, its gradient survives many steps too. The spec lstm_single.t27 computes the cell.
+
Try it
Load Remember and then Forget and compare the new memory; then write with the input gate open.
Set the biases of the forget, input and output gates and the old memory: the new memory is forget x old + input x candidate, and the output shows tanh of it through the output gate.
+
specs/ml/recurrent/lstm_single.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// LSTM solves vanishing gradient problem | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: called `@sigmoid` and `@tanh`, which Zig does not have; each gate weight
+// was one number per INPUT, indexed `i % hidden_size`, so it was not an LSTM; and
+// every test asserted `true`. Now each gate has its own matrices, W_x
+// (hidden x input) and W_h (hidden x hidden), and a bias:
+// i = s(z_i), f = s(z_f), g = tanh(z_g), o = s(z_o), z = W_x x + W_h h + b
+// c' = f c + i g, h' = o tanh(c')
+// backward gives every gradient of one step, and the tests check it against finite
+// differences.
+
+module LstmCell;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const FORGET_BIAS_INIT : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const LSTMConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ };
+
+ pub const LSTMState = struct {
+ h : []f32,
+ c : []f32,
+ };
+
+ pub const LSTMWeights = struct {
+ W_ii : []f32,
+ W_hi : []f32,
+ b_i : []f32,
+ W_if : []f32,
+ W_hf : []f32,
+ b_f : []f32,
+ W_ig : []f32,
+ W_hg : []f32,
+ b_g : []f32,
+ W_io : []f32,
+ W_ho : []f32,
+ b_o : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Pre-activation of one gate for hidden unit u: W_x[u] . x + W_h[u] . h + b[u]
+ fn pre(w_x: []const f32, w_h: []const f32, b: []const f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig) -> f32 {
+ var z : f32 = b[u];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + w_x[u * config.input_size + j] * x[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + w_h[u * config.hidden_size + k] * h[k];
+ k = k + 1;
+ }
+ return z;
+ }
+
+ fn forward(input: []const f32, state: LSTMState, weights: LSTMWeights, output_state: LSTMState, config: LSTMConfig) -> void {
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ output_state.c[u] = fg * state.c[u] + ig * gg;
+ output_state.h[u] = og * tanh(output_state.c[u]);
+ u = u + 1;
+ }
+ }
+
+ // Adds dz * (x, h, 1) into one gate's weight gradients, and dz * W into grad_input / grad_state.h
+ fn spread(dz: f32, w_x: []const f32, w_h: []const f32, gw_x: []f32, gw_h: []f32, gb: []f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig, grad_input: []f32, grad_h: []f32) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ gw_x[u * config.input_size + j] = gw_x[u * config.input_size + j] + dz * x[j];
+ grad_input[j] = grad_input[j] + dz * w_x[u * config.input_size + j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ gw_h[u * config.hidden_size + k] = gw_h[u * config.hidden_size + k] + dz * h[k];
+ grad_h[k] = grad_h[k] + dz * w_h[u * config.hidden_size + k];
+ k = k + 1;
+ }
+ gb[u] = gb[u] + dz;
+ }
+
+ // One step back. grad_output = dL/dh', grad_cell = dL/dc' from later steps. Writes
+ // dL/dx, dL/dh, dL/dc and ADDS the weight gradients into grad_weights.
+ // do = dh' tanh(c'); dc = dh' o (1 - tanh^2 c') + grad_cell
+ // dz_o = do o (1 - o), dz_f = dc c f (1 - f), dz_i = dc g i (1 - i), dz_g = dc i (1 - g^2)
+ // dL/dc = dc f
+ fn backward(grad_output: []const f32, grad_cell: []const f32, input: []const f32, state: LSTMState, weights: LSTMWeights, config: LSTMConfig, grad_input: []f32, grad_state: LSTMState, grad_weights: LSTMWeights) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ grad_input[j] = 0.0;
+ j = j + 1;
+ }
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ grad_state.h[u] = 0.0;
+ u = u + 1;
+ }
+ u = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ const c_new = fg * state.c[u] + ig * gg;
+ const tc = tanh(c_new);
+ const dc = grad_output[u] * og * (1.0 - tc * tc) + grad_cell[u];
+ const dz_o = grad_output[u] * tc * og * (1.0 - og);
+ const dz_f = dc * state.c[u] * fg * (1.0 - fg);
+ const dz_i = dc * gg * ig * (1.0 - ig);
+ const dz_g = dc * ig * (1.0 - gg * gg);
+ grad_state.c[u] = dc * fg;
+ spread(dz_i, weights.W_ii, weights.W_hi, grad_weights.W_ii, grad_weights.W_hi, grad_weights.b_i, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_f, weights.W_if, weights.W_hf, grad_weights.W_if, grad_weights.W_hf, grad_weights.b_f, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_g, weights.W_ig, weights.W_hg, grad_weights.W_ig, grad_weights.W_hg, grad_weights.b_g, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_o, weights.W_io, weights.W_ho, grad_weights.W_io, grad_weights.W_ho, grad_weights.b_o, input, state.h, u, config, grad_input, grad_state.h);
+ u = u + 1;
+ }
+ }
+
+ // The fixture: input 3, hidden 2. All 48 weights live in one buffer, per gate
+ // W_x (6), W_h (4), b (2), in the gate order i, f, g, o.
+ fn cfg() -> LSTMConfig {
+ return LSTMConfig{ .input_size = 3, .hidden_size = 2 };
+ }
+
+ fn weights_of(w: []f32) -> LSTMWeights {
+ return LSTMWeights{
+ .W_ii = w[0..6], .W_hi = w[6..10], .b_i = w[10..12],
+ .W_if = w[12..18], .W_hf = w[18..22], .b_f = w[22..24],
+ .W_ig = w[24..30], .W_hg = w[30..34], .b_g = w[34..36],
+ .W_io = w[36..42], .W_ho = w[42..46], .b_o = w[46..48],
+ };
+ }
+
+ // Deterministic weights in [-0.5, 0.5]; the forget biases start at FORGET_BIAS_INIT
+ fn fill(w: []f32) -> void {
+ var k : usize = 0;
+ while (k < 48) {
+ w[k] = @as(f32, @floatFromInt((k * 7 + 3) % 11)) / 10.0 - 0.5;
+ k = k + 1;
+ }
+ w[22] = FORGET_BIAS_INIT;
+ w[23] = FORGET_BIAS_INIT;
+ }
+
+ // L = 0.5 h'_0 - 0.3 h'_1 + 0.2 c'_0 + 0.1 c'_1 after one step, with one input, one h or
+ // one c entry moved by `delta` (which: 0 = none, 1 + j = x_j, 4 + k = h_k, 6 + k = c_k,
+ // 8 = b_f[1])
+ fn loss(which: usize, delta: f32) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ var x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ if (which >= 1 and which <= 3) {
+ x[which - 1] = x[which - 1] + delta;
+ }
+ if (which >= 4 and which <= 5) {
+ h[which - 4] = h[which - 4] + delta;
+ }
+ if (which >= 6 and which <= 7) {
+ c[which - 6] = c[which - 6] + delta;
+ }
+ if (which == 8) {
+ w[23] = w[23] + delta;
+ }
+ var h_out : [2]f32 = [0.0] ** 2;
+ var c_out : [2]f32 = [0.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ return 0.5 * h_out[0] - 0.3 * h_out[1] + 0.2 * c_out[0] + 0.1 * c_out[1];
+ }
+
+ // The analytic gradient of that loss, same `which` numbering
+ fn grad(which: usize) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const grad_output : [2]f32 = [0.5, -0.3];
+ const grad_cell : [2]f32 = [0.2, 0.1];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&grad_output, &grad_cell, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ if (which >= 1 and which <= 3) {
+ return gx[which - 1];
+ }
+ if (which >= 4 and which <= 5) {
+ return gh[which - 4];
+ }
+ if (which >= 6 and which <= 7) {
+ return gc[which - 6];
+ }
+ return gw[23];
+ }
+
+ // Sum of |every gradient| when grad_output and grad_cell are both 0
+ fn zero_upstream_total() -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const zeros : [2]f32 = [0.0, 0.0];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&zeros, &zeros, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ var total : f32 = @abs(gx[0]) + @abs(gx[1]) + @abs(gx[2]) + @abs(gh[0]) + @abs(gh[1]) + @abs(gc[0]) + @abs(gc[1]);
+ var k : usize = 0;
+ while (k < 48) {
+ total = total + @abs(gw[k]);
+ k = k + 1;
+ }
+ return total;
+ }
+
+ fn slope(which: usize) -> f32 {
+ return (loss(which, 0.001) - loss(which, -0.001)) / 0.002;
+ }
+
+ // which: 0 = h'_0, 1 = c'_0 for a 1-unit cell with every weight 0 and the given biases
+ fn bias_only(b_i: f32, b_f: f32, b_g: f32, b_o: f32, c_prev: f32, which: u8) -> f32 {
+ var w : [48]f32 = [0.0] ** 48;
+ w[10] = b_i;
+ w[22] = b_f;
+ w[34] = b_g;
+ w[46] = b_o;
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.0, 0.0];
+ var c : [2]f32 = [c_prev, 0.0];
+ var h_out : [2]f32 = [9.0] ** 2;
+ var c_out : [2]f32 = [9.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ if (which == 0) {
+ return h_out[0];
+ }
+ return c_out[0];
+ }
+
+ fn valid_input() -> usize {
+ return cfg().input_size;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // All weights 0: every gate is s(0) = 0.5 or tanh(0) = 0, so c' = 0.5 c and h' = 0.5 tanh(c')
+ test forward_basic_case
+ given c_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 1)
+ when h_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 0)
+ then @abs(c_new - 0.4) < 0.000001 and @abs(h_new - 0.5 * tanh(c_new)) < 0.000001 and @abs(h_new - 0.1899745) < 0.000001
+
+ // A closed forget gate (b_f = -30) erases the old cell; an open one (b_f = 30) keeps it
+ test forward_basic_lstm
+ given erased = bias_only(0.0, -30.0, 0.0, 0.0, 0.8, 1)
+ when kept = bias_only(-30.0, 30.0, 0.0, 0.0, 0.8, 1)
+ then @abs(erased) < 0.000001 and @abs(kept - 0.8) < 0.000001
+
+ // A closed input gate and forget gate leave c' = 0, so h' = 0 whatever the output gate does
+ test forward_with_empty_input
+ given h_new = bias_only(-30.0, -30.0, 1.0, 30.0, 0.8, 0)
+ then @abs(h_new) < 0.000001
+
+ // dL/dx_j against central differences
+ test backward_basic_case
+ given g0 = grad(1)
+ when g2 = grad(3)
+ then @abs(g0 - slope(1)) < 0.0005 and @abs(grad(2) - slope(2)) < 0.0005 and @abs(g2 - slope(3)) < 0.0005
+
+ // dL/dh_k, dL/dc_k and dL/db_f[1] against central differences
+ test backward_basic_lstm
+ given gh0 = grad(4)
+ when gc1 = grad(7)
+ then @abs(gh0 - slope(4)) < 0.0005 and @abs(grad(5) - slope(5)) < 0.0005 and @abs(grad(6) - slope(6)) < 0.0005 and @abs(gc1 - slope(7)) < 0.0005 and @abs(grad(8) - slope(8)) < 0.0005
+
+ // With no upstream gradient nothing flows back
+ test backward_with_empty_grad
+ given total = zero_upstream_total()
+ then total == 0.0 and @abs(grad(1)) > 0.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.size() == config.input_size
+ invariant lstm_cell_constraint_0
+ assert valid_input() == 3
+
+ // state.h.size() == config.hidden_size
+ invariant lstm_cell_constraint_1
+ assert cfg().hidden_size == 2
+
+ // state.c.size() == config.hidden_size; the forget bias starts open
+ invariant lstm_cell_constraint_2
+ assert FORGET_BIAS_INIT > 0.0
+
What max pooling keeps, where it remembers taking it from, and where the gradient goes.
+
Max pooling keeps the largest value of each window and drops the rest, so the network cares less about exactly where a feature sits. On the spec's 4 x 4 grid, 2 x 2 windows with stride 2 keep 4, 8, 9 and -2, taken from cells 5, 6, 13 and 11. Overlapping 3 x 3 windows with stride 1 keep 8, 8, 9 and 9. In the backward pass each window's whole gradient goes to its maximum: gradients 1, 2, 3 and 4 land on cells 5, 6, 13 and 11, and every other cell gets 0. The spec maxpool2d_layer.t27 computes the outputs, their positions and the gradient.
+
Try it
Switch to 3 x 3 windows and find the cell two outputs share; then turn on the gradient for 2 x 2 windows.
Pick a window size and stride and tap an output: max pooling keeps the largest value of its window and remembers where it was, and the gradient later goes to that one cell.
+
specs/ml/layers/maxpool2d_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Reduce spatial dimensions | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward and backward compared two values and threw them away
+// (placeholders), and every test asserted `result != undefined`. Max pooling now
+// runs over one H x W channel (row-major) with a k x k window and a stride. It
+// records each window's argmax, and backward sends each gradient back to that
+// element only. Padding and dilation are carried in the config but not read: the
+// layer assumes padding 0 and dilation 1.
+
+module Maxpool2d;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const MaxPool2DConfig = struct {
+ kernel_size : u32,
+ stride : u32,
+ padding : u32,
+ dilation : u32,
+ return_indices : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Output side for an input side n: (n - k) / stride + 1
+ fn out_size(n: u32, config: MaxPool2DConfig) -> u32 {
+ return (n - config.kernel_size) / config.stride + 1;
+ }
+
+ // output[oy][ox] = max of the k x k window at (oy * stride, ox * stride);
+ // indices[oy][ox] = the flat input index that held it
+ fn forward(input: []const f32, height: u32, width: u32, config: MaxPool2DConfig, output: []f32, indices: []u32) -> void {
+ const oh = out_size(height, config);
+ const ow = out_size(width, config);
+ var oy : u32 = 0;
+ while (oy < oh) {
+ var ox : u32 = 0;
+ while (ox < ow) {
+ var best_index : u32 = (oy * config.stride) * width + ox * config.stride;
+ var ky : u32 = 0;
+ while (ky < config.kernel_size) {
+ var kx : u32 = 0;
+ while (kx < config.kernel_size) {
+ const idx : u32 = (oy * config.stride + ky) * width + ox * config.stride + kx;
+ if (input[idx] > input[best_index]) {
+ best_index = idx;
+ }
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * ow + ox] = input[best_index];
+ indices[oy * ow + ox] = best_index;
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ }
+
+ // grad_input[indices[o]] += grad_output[o]; every other input gets 0
+ fn backward(grad_output: []const f32, indices: []const u32, grad_input: []f32) -> void {
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var o : usize = 0;
+ while (o < grad_output.len) {
+ grad_input[indices[o]] = grad_input[indices[o]] + grad_output[o];
+ o = o + 1;
+ }
+ }
+
+ fn cfg(kernel: u32, stride: u32) -> MaxPool2DConfig {
+ return MaxPool2DConfig{ .kernel_size = kernel, .stride = stride, .padding = 0, .dilation = 1, .return_indices = true };
+ }
+
+ // The output side on the 4 x 4 fixture for a k x k window and a stride.
+ // out_size takes a MaxPool2DConfig struct and does not lower to WASM; this
+ // takes two integers, so a host reads the side from the compiled spec.
+ fn out_side(kernel: u32, stride: u32) -> u32 {
+ return out_size(4, cfg(kernel, stride));
+ }
+
+ // The 4 x 4 fixture
+ // 1 2 5 6
+ // 3 4 8 7
+ // 0 -1 -3 -2
+ // -4 9 -6 -5
+ fn pool_at(kernel: u32, stride: u32, which: u8, index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [9]f32 = [0.0] ** 9;
+ var indices : [9]u32 = [0] ** 9;
+ forward(&input, 4, 4, cfg(kernel, stride), &output, &indices);
+ if (which == 0) {
+ return output[index];
+ }
+ return @as(f32, @floatFromInt(indices[index]));
+ }
+
+ // 2 x 2, stride 2, gradients (1, 2, 3, 4) for the four windows; grad_input[index]
+ fn backward_at(index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [4]f32 = [0.0] ** 4;
+ var indices : [4]u32 = [0] ** 4;
+ forward(&input, 4, 4, cfg(2, 2), &output, &indices);
+ const grad_output : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, &indices, &grad_input);
+ return grad_input[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // 2 x 2, stride 2: windows' maxima are 4, 8, 9, -2
+ test forward_basic_case
+ given first = pool_at(2, 2, 0, 0)
+ when last = pool_at(2, 2, 0, 3)
+ then first == 4.0 and pool_at(2, 2, 0, 1) == 8.0 and pool_at(2, 2, 0, 2) == 9.0 and last == -2.0
+
+ // The argmax indices: 4 at 5, 8 at 6, 9 at 13, -2 at 11
+ test forward_max_finding
+ given first = pool_at(2, 2, 1, 0)
+ when third = pool_at(2, 2, 1, 2)
+ then first == 5.0 and pool_at(2, 2, 1, 1) == 6.0 and third == 13.0 and pool_at(2, 2, 1, 3) == 11.0
+
+ // 3 x 3, stride 1 gives a 2 x 2 output whose windows overlap
+ test forward_overlapping_windows
+ given first = pool_at(3, 1, 0, 0)
+ when last = pool_at(3, 1, 0, 3)
+ then first == 8.0 and pool_at(3, 1, 0, 1) == 8.0 and pool_at(3, 1, 0, 2) == 9.0 and last == 9.0
+
+ test backward_basic_case
+ given at_max = backward_at(5)
+ when elsewhere = backward_at(0)
+ then at_max == 1.0 and elsewhere == 0.0
+
+ // Each window's gradient lands on its argmax only: the sum is conserved
+ test backward_gradient_propagation
+ given total = backward_at(5) + backward_at(6) + backward_at(13) + backward_at(11)
+ when others = backward_at(0) + backward_at(1) + backward_at(4) + backward_at(15)
+ then total == 10.0 and others == 0.0 and backward_at(13) == 3.0
+
+ // The four window/stride pairs the nn-maxpool widget offers, on the 4 x 4 fixture:
+ // 2 x 2 stride 2 -> 2, 3 x 3 stride 1 -> 2, 2 x 2 stride 1 -> 3, 4 x 4 stride 1 -> 1
+ test out_side_of_each_widget_config
+ given halved = out_side(2, 2)
+ when whole = out_side(4, 1)
+ then halved == 2 and out_side(3, 1) == 2 and out_side(2, 1) == 3 and whole == 1
+
+ // Edges: a 1 x 1 window at stride 1 keeps the side; a stride that does not
+ // divide (4 - k) rounds down (3 x 3 at stride 2 fits one window, 1 x 1 at
+ // stride 3 fits two)
+ test out_side_identity_and_floor
+ given same = out_side(1, 1)
+ when floored = out_side(3, 2)
+ then same == 4 and floored == 1 and out_side(1, 3) == 2
+
+ invariant maxpool_output_size_formula
+ assert out_size(4, cfg(2, 2)) == 2 and out_size(4, cfg(3, 1)) == 2 and out_size(5, cfg(2, 2)) == 2
+
How MSE turns every miss into one number, why it chases large errors, and where its gradient points.
+
A loss turns all the misses of a model into one number to push down. Mean squared error averages the squared misses: predictions 1.5, 2 and 2 against targets 1, 2 and 3 cost (0.25 + 0 + 1) / 3 = 0.4167. Squaring makes a miss of 2 cost 4 times a miss of 1, so MSE chases large errors. Its gradient for each prediction is 2 (p - t) / n, here 0.333, 0 and -0.667: each prediction is pushed back toward its target. As a function of one prediction the loss is a parabola, a bowl with one bottom. The spec mse_loss.t27 computes the loss and the gradient.
+
Try it
Move prediction 2 onto its target and read the loss; then drag along the bowl and find its bottom.
Move three predictions against the targets 1, 2 and 3: the loss is the mean of the squared misses, its bowl is a parabola, and each gradient points its prediction back at the target.
+
specs/ml/loss/mse_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Penalizes large errors more heavily | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/mse_loss.tri: the port kept only
+// the first parameter and returned void, so forward computed a value and
+// threw it away, and every test asserted `result != undefined`.
+
+module MseLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ // reduction: "mean", "sum" or "none" (from the .tri). forward below is the
+ // .tri's mean; the field is carried, not yet read.
+ pub const MSELossConfig = struct {
+ reduction : []const u8,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(predictions, targets) -> mean((predictions - targets)^2); 0 for n = 0
+ fn forward(predictions: []const f32, targets: []const f32) -> f32 {
+ if (predictions.len == 0) {
+ return 0.0;
+ }
+ var sum_loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ const diff = predictions[i] - targets[i];
+ sum_loss = sum_loss + diff * diff;
+ i = i + 1;
+ }
+ return sum_loss / (predictions.len as f32);
+ }
+
+ // backward: grad_input[i] = 2 (predictions[i] - targets[i]) / n
+ fn backward(predictions: []const f32, targets: []const f32, grad_input: []f32) -> void {
+ if (predictions.len == 0) {
+ return;
+ }
+ const scale = 2.0 / (predictions.len as f32);
+ var i : usize = 0;
+ while (i < predictions.len) {
+ grad_input[i] = scale * (predictions[i] - targets[i]);
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ fn mse_at(p0: f32, p1: f32, p2: f32) -> f32 {
+ const predictions : [3]f32 = [p0, p1, p2];
+ const targets : [3]f32 = [1.0, 2.0, 3.0];
+ return forward(&predictions, &targets);
+ }
+
+ fn mse_grad_at(p0: f32, p1: f32, p2: f32, index: usize) -> f32 {
+ const predictions : [3]f32 = [p0, p1, p2];
+ const targets : [3]f32 = [1.0, 2.0, 3.0];
+ var grads : [3]f32 = [0.0] ** 3;
+ backward(&predictions, &targets, &grads);
+ return grads[index];
+ }
+
+ fn mse_empty() -> f32 {
+ const none : [0]f32 = [];
+ return forward(&none, &none);
+ }
+
+ test forward_zero_on_exact_match
+ given result = mse_at(1.0, 2.0, 3.0)
+ when also = mse_empty()
+ then result == 0.0 and also == 0.0
+
+ // errors (0.5, -1, 2): (0.25 + 1 + 4) / 3 = 1.75
+ test forward_basic_case
+ given result = mse_at(1.5, 1.0, 5.0)
+ when flipped = mse_at(0.5, 3.0, 1.0)
+ then @abs(result - 1.75) < 0.000001 and @abs(flipped - result) < 0.000001
+
+ test backward_matches_finite_difference
+ given slope = (mse_at(1.5, 1.001, 5.0) - mse_at(1.5, 0.999, 5.0)) / 0.002
+ when result = mse_grad_at(1.5, 1.0, 5.0, 1)
+ then @abs(result - slope) < 0.001 and @abs(result - (-2.0 / 3.0)) < 0.000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant mse_zero_at_target
+ assert mse_at(1.0, 2.0, 3.0) == 0.0
+
+
+
+
+
+
diff --git a/apps/website/public/learn/neural-networks.t27 b/apps/website/public/learn/neural-networks.t27
new file mode 100644
index 0000000000..1d004ddf19
--- /dev/null
+++ b/apps/website/public/learn/neural-networks.t27
@@ -0,0 +1,380 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/course/neural-networks.t27 -- the t27 course on neural networks, from one neuron to a transformer block
+; Source of truth for apps/website/scripts/course-from-spec.mjs (gHashTag/trinity), which reads
+; it the way it reads specs/course/course.t27: the constant schema, every test block below, every
+; widget in specs/widgets/gallery.t27, every lesson spec compiled clean, the Russian bundle named
+; by specs/course/neural-networks-ru.t27. It writes this course into src/lib/course.generated.ts
+; (the page at #/neural-networks) and a byte-identical copy of this file at
+; public/learn/neural-networks.t27. ASCII only (L3), English only (LANG-EN).
+
+; WHAT THE COURSE IS: course 10 of specs/course/courses.t27, 27 lessons in 9 modules of 3: one
+; neuron, new bends, a layer, pictures, the error, downhill, learning, memory and attention, and
+; the transformer and the reward. Every lesson opens one interactive widget and one spec. The
+; agents of this project think with neural networks; this course opens their parts as the
+; gHashTag/t27 specs/ml specs they are written in, each with numeric tests. Every widget runs the
+; fn bodies of one of those specs compiled to WebAssembly (public/widgets//logic.js, held to
+; its spec by scripts/widget-logic.mjs), vendored byte for byte from t27 master 239b4b059; the
+; max-pool spec from 2e53089d5, which added out_side (gHashTag/t27#10665) for the nn-maxpool widget.
+
+; WHAT THE COURSE DOES NOT DO: it says no number of its own. Every number in a lesson is one the
+; lesson spec computes and its widget shows. Lesson 10 is average pooling, not a convolution: the
+; sliding conv2d spec (gHashTag/t27#10612) was not merged when the course was made, and the
+; lesson says so. Lesson 20 shows the four inputs of XOR, not a decision region over the plane.
+
+; WHY 27: 3 cubed lessons, the size of a TRI-27 word. Changing the shape changes the tests.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module neural_networks;
+
+pub const KIND : str = "course";
+pub const ID : str = "neural-networks";
+pub const SCHEMA_VERSION : u8 = 1;
+; The derived files the generator rewrites (relative to apps/website).
+pub const GENERATED : [2]str = ["src/lib/course.generated.ts", "public/learn/neural-networks.t27"];
+pub const ROUTE : str = "neural-networks";
+; Share pages: learn/ for course 1, learn// for the next (specs/course/courses.t27).
+pub const SHARE_PATH : str = "learn/neural-networks/";
+; The paid training every course page links to (specs/course/arithmetic-and-dsp.t27).
+pub const COHORT_ROUTE : str = "fpga-training";
+pub const GALLERY : str = "specs/widgets/gallery.t27";
+pub const LOCALES : [2]str = ["en", "ru"];
+pub const RU_CONTRACT : str = "specs/course/neural-networks-ru.t27";
+; Lesson marks the reader sets are kept in this browser only, under this key, one key for
+; every course.
+pub const PROGRESS_KEY : str = "t27-course-done";
+pub const SENDS_NOTHING : bool = true;
+; A lesson's widget is the page's main stage, so it is drawn taller than a
+; gallery card (specs/widgets/gallery.t27 EMBED_* heights are for embeds in posts).
+pub const TOOL_FRAME_HEIGHT : u16 = 760;
+pub const PLAYER_FRAME_HEIGHT : u16 = 480;
+
+; --- Words on the page ---------------------------------------------------------------------
+pub const TITLE : str = "Neural networks in t27: from one neuron to a transformer block";
+pub const DESCRIPTION : str = "Course 10, 27 lessons, a live widget and a t27 spec each: activations, softmax and embeddings, pooling and batch norm, losses, SGD, RMSProp and Adam, backpropagation and XOR, recurrent cells, attention, a transformer block and PPO. Every number is computed by a compiled spec.";
+pub const SAY_KICKER : str = "Course";
+pub const SAY_LEAD : str = "The agents of this project think with neural networks, and here their parts are written as t27 specs with tests, specs the compiler checks. Every widget runs the fn bodies of a specs/ml spec compiled to WebAssembly: move a slider and the spec answers.";
+pub const SAY_SHAPE : str = "9 modules of 3 lessons, 27 cells. A filled cell is a lesson you marked done.";
+pub const SAY_START : str = "Start lesson 1";
+pub const SAY_CONTINUE : str = "Continue";
+pub const SAY_MODULE : str = "Module {0}";
+pub const SAY_LESSON : str = "Lesson {0} of {1}";
+pub const SAY_GOAL : str = "You will learn";
+pub const SAY_TRY : str = "Try it";
+pub const SAY_ALSO : str = "Also try";
+pub const SAY_ALSO_HINT : str = "Each one swaps the widget above.";
+pub const SAY_BACK_TO_MAIN : str = "Back to this lesson's widget";
+pub const SAY_SPEC : str = "Open the lesson's spec in the player";
+pub const SAY_OPEN_PAGE : str = "Open the widget on its own page";
+pub const SAY_WIDGET_LANG : str = "Widgets keep their own English words: each one lives in its own t27 spec.";
+pub const SAY_PREV : str = "Previous";
+pub const SAY_NEXT : str = "Next";
+pub const SAY_ALL : str = "All lessons";
+pub const SAY_MARK : str = "Mark as done";
+pub const SAY_MARKED : str = "Done";
+pub const SAY_PROGRESS : str = "{0} of {1} done";
+pub const SAY_PRIVATE : str = "Progress stays in this browser; nothing is sent.";
+pub const SAY_SOURCE : str = "This course is itself a t27 spec: read it";
+pub const SAY_COHORT : str = "Looking for the paid FPGA training? It moved to its own page.";
+pub const SAY_NOT_FOUND : str = "No lesson has this address.";
+pub const SAY_OPEN_LESSON : str = "Open the interactive lesson";
+pub const SAY_OPEN_COURSE : str = "Open the interactive course";
+pub const SAY_SEO_TITLE : str = "{0}: neural networks course, lesson {1} of {2}";
+pub const SAY_SHARE : str = "Link to share, with a preview card";
+pub const SAY_NEXT_COURSE : str = "Next course";
+pub const SAY_PREV_COURSE : str = "Previous course";
+
+; --- Modules -------------------------------------------------------------------------------
+pub const LESSONS_PER_MODULE : u8 = 3;
+pub const MODULE_COUNT : u8 = 9;
+pub const MODULE_IDS : [9]str = ["one-neuron", "new-bends", "a-layer", "pictures", "the-error", "downhill", "learning", "memory-and-attention", "transformer-and-reward"];
+pub const MODULE_TITLES : [9]str = [
+ "One neuron",
+ "New bends",
+ "A layer",
+ "Pictures",
+ "The error",
+ "Downhill",
+ "Learning",
+ "Memory and attention",
+ "The transformer and the reward"
+];
+pub const MODULE_LINES : [9]str = [
+ "A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back.",
+ "The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff.",
+ "What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off.",
+ "Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales.",
+ "How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers.",
+ "Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them.",
+ "Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches.",
+ "Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once.",
+ "The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching."
+];
+
+; --- Lessons -------------------------------------------------------------------------------
+; LESSON_WIDGETS names the widget the lesson opens; every one is a gallery id, and no two
+; lessons of any course open the same one. Each module's three widgets are offered in each
+; other's LESSON_ALSO, next to related widgets of this course and of the ternary course.
+; LESSON_SPECS is the spec under public/t27/files/ the lesson opens in the player, and the spec
+; whose compiled fn bodies the widget runs.
+pub const LESSON_COUNT : u8 = 27;
+pub const LESSON_IDS : [27]str = [
+ "relu", "the-sigmoid", "tanh-and-saturation",
+ "gelu", "swish-and-silu", "elu",
+ "softmax-and-temperature", "embeddings", "dropout",
+ "patches-of-a-picture", "max-pooling", "batch-normalisation",
+ "mean-squared-error", "cross-entropy", "the-huber-loss",
+ "gradient-descent", "rmsprop", "adam",
+ "backpropagation", "learning-xor", "lamb-and-large-batches",
+ "a-recurrent-cell", "lstm-gates", "attention",
+ "positional-encoding", "an-encoder-block", "ppo-and-the-reward"
+];
+pub const LESSON_MODULES : [27]str = [
+ "one-neuron", "one-neuron", "one-neuron",
+ "new-bends", "new-bends", "new-bends",
+ "a-layer", "a-layer", "a-layer",
+ "pictures", "pictures", "pictures",
+ "the-error", "the-error", "the-error",
+ "downhill", "downhill", "downhill",
+ "learning", "learning", "learning",
+ "memory-and-attention", "memory-and-attention", "memory-and-attention",
+ "transformer-and-reward", "transformer-and-reward", "transformer-and-reward"
+];
+pub const LESSON_WIDGETS : [27]str = [
+ "nn-relu", "nn-sigmoid", "nn-tanh",
+ "nn-gelu", "nn-silu", "nn-elu",
+ "nn-softmax", "nn-embedding", "nn-dropout",
+ "nn-conv", "nn-maxpool", "nn-batchnorm",
+ "nn-mse", "nn-bce", "nn-huber",
+ "nn-sgd", "nn-rmsprop", "nn-adam",
+ "nn-backprop", "nn-xor", "nn-lamb",
+ "nn-rnn", "nn-lstm", "nn-attention",
+ "nn-positions", "nn-encoder", "nn-ppo"
+];
+pub const LESSON_ALSO : [27]str = [
+ "nn-sigmoid,nn-tanh,tc-one-neuron",
+ "nn-relu,nn-tanh,nn-bce",
+ "nn-relu,nn-sigmoid",
+ "nn-silu,nn-elu,nn-encoder",
+ "nn-gelu,nn-elu,nn-relu",
+ "nn-relu,nn-silu,nn-gelu",
+ "nn-embedding,nn-dropout,nn-bce",
+ "nn-softmax,nn-dropout,nn-positions",
+ "nn-softmax,nn-embedding",
+ "nn-maxpool,nn-batchnorm,tc-detector",
+ "nn-conv,nn-batchnorm",
+ "nn-conv,nn-maxpool,nn-encoder",
+ "nn-bce,nn-huber",
+ "nn-mse,nn-huber,nn-sigmoid",
+ "nn-mse,nn-bce",
+ "nn-rmsprop,nn-adam,nn-mse",
+ "nn-sgd,nn-adam",
+ "nn-sgd,nn-rmsprop,nn-lamb",
+ "nn-xor,nn-lamb,tc-tiny-layer",
+ "nn-backprop,nn-lamb,tc-tiny-layer",
+ "nn-adam,nn-xor,nn-backprop",
+ "nn-lstm,nn-attention",
+ "nn-rnn,nn-attention,nn-sigmoid",
+ "nn-rnn,nn-lstm,nn-encoder",
+ "nn-attention,nn-encoder,nn-embedding",
+ "nn-attention,nn-positions,nn-batchnorm",
+ "nn-softmax,nn-adam,nn-encoder"
+];
+pub const LESSON_SPECS : [27]str = [
+ "specs/ml/activation/relu_activation.t27",
+ "specs/ml/activation/sigmoid_activation.t27",
+ "specs/ml/activation/tanh_activation.t27",
+ "specs/ml/activation/gelu_activation.t27",
+ "specs/ml/activation/silu_swish_activation.t27",
+ "specs/ml/activation/elu_activation.t27",
+ "specs/ml/activation/softmax.t27",
+ "specs/ml/layers/embedding_layer.t27",
+ "specs/ml/layers/dropout_layer.t27",
+ "specs/ml/layers/avgpool2d_layer.t27",
+ "specs/ml/layers/maxpool2d_layer.t27",
+ "specs/ml/layers/batchnorm_layer.t27",
+ "specs/ml/loss/mse_loss.t27",
+ "specs/ml/loss/binary_crossentropy_loss.t27",
+ "specs/ml/loss/huber_loss.t27",
+ "specs/ml/optimizer/sgd.t27",
+ "specs/ml/optimizer/rmsprop.t27",
+ "specs/ml/optimizer/adam.t27",
+ "specs/ml/layers/dense_layer.t27",
+ "specs/ml/pathway/mlp.t27",
+ "specs/ml/optimizer/lamb.t27",
+ "specs/ml/recurrent/rnn_cell.t27",
+ "specs/ml/recurrent/lstm_single.t27",
+ "specs/ml/recurrent/attention_mechanism.t27",
+ "specs/ml/transformer/positional_encoding.t27",
+ "specs/ml/transformer/encoder_block.t27",
+ "specs/ml/rl/ppo_clip_loss.t27"
+];
+pub const LESSON_TITLES : [27]str = [
+ "ReLU: keep the positive part",
+ "The sigmoid: a number into a probability",
+ "tanh and saturation",
+ "GELU: a dip below zero",
+ "Swish and SiLU",
+ "ELU: a floor, not a cliff",
+ "Softmax and temperature",
+ "Embeddings",
+ "Dropout",
+ "Patches of a picture",
+ "Max pooling",
+ "Batch normalisation",
+ "Mean squared error",
+ "Cross-entropy",
+ "The Huber loss",
+ "Gradient descent",
+ "RMSProp",
+ "Adam",
+ "Backpropagation",
+ "Learning XOR",
+ "LAMB and large batches",
+ "A recurrent cell",
+ "LSTM gates",
+ "Attention",
+ "Positional encoding",
+ "An encoder block",
+ "PPO and the reward"
+];
+pub const LESSON_GOALS : [27]str = [
+ "What ReLU does to a positive and to a negative input, and why a slope of 0 stops a unit from learning.",
+ "How the sigmoid maps any input to a probability, and why its slope limits deep networks.",
+ "Why tanh is centred on zero, and where its slope falls so low that a unit stops learning.",
+ "What GELU computes, why it dips below zero, and how close its tanh shortcut comes.",
+ "How x times sigmoid(beta x) gates itself, and how one parameter moves it from a smooth curve to ReLU.",
+ "How ELU keeps a soft negative side with a floor at -alpha, and why its slope never quite dies.",
+ "How softmax turns scores into probabilities that sum to one, and what the temperature does to them.",
+ "How a token becomes a row of numbers, and why learning changes only the rows that were read.",
+ "Why switching units off while training helps, and why the survivors must be scaled.",
+ "Why networks read a picture in patches, and what a 2 x 2 window averaged over each patch gives.",
+ "What max pooling keeps, where it remembers taking it from, and where the gradient goes.",
+ "How batch norm puts features on different scales on one scale, and what it uses at inference.",
+ "How MSE turns every miss into one number, why it chases large errors, and where its gradient points.",
+ "Why cross-entropy charges a confident wrong answer so much, and why that is what trains a classifier.",
+ "How the Huber loss treats a small error like squared error and a large one linearly, so outliers pull less.",
+ "How gradient descent walks down a loss, what momentum adds, and why too large a rate never settles.",
+ "How RMSProp divides each step by a running root mean square, so the step no longer depends on the gradient's size.",
+ "How Adam combines momentum with a running scale, and why its bias correction makes step 1 a fixed size.",
+ "How the gradient of each weight and each input of a layer follows from the gradient of its outputs.",
+ "Why one neuron cannot learn XOR, and how a hidden layer trained by backpropagation does.",
+ "How LAMB scales each layer's step by its own weight norm, so one learning rate suits layers of every size.",
+ "How a recurrent cell carries a state from step to step, and why that memory fades.",
+ "How the forget, input and output gates of an LSTM decide what the cell keeps, writes and shows.",
+ "How attention scores every query against every key, what a softmax over each row gives, and what the causal mask removes.",
+ "How sines and cosines at falling frequencies give every position its own vector, and why a shift is a turn.",
+ "What one transformer encoder block does to its input, and what moving the norms in front changes.",
+ "How PPO turns a reward into a policy update, and how its clip keeps one update from going too far."
+];
+pub const LESSON_TEXTS : [27]str = [
+ "A neuron adds up its weighted inputs and passes the sum through an activation. ReLU is the plainest one, max(0, x): a positive sum goes through unchanged and a negative one becomes 0. Its slope is 1 or 0, and a slope of 0 is a unit that has stopped learning, because no gradient flows back through it. A leaky slope fixes that: with a slope of 0.1, x = -2 gives -0.2 and passes back a gradient of 0.1. The spec relu_activation.t27 computes both curves the widget draws.",
+ "The sigmoid 1 / (1 + e^-x) squeezes the whole number line into 0 to 1, which is why it reads as a probability: 2 becomes 0.8808 and -4 becomes 0.018. Its slope is s (1 - s), computed from the output s itself, and it is largest at 0, where it is 0.25; at x = 8 it is 0.000335. A gradient that passes back through a stack of sigmoids is multiplied by at most 0.25 at each one, which is why deep sigmoid networks trained slowly. The spec sigmoid_activation.t27 computes the curve and the slope.",
+ "tanh is a sigmoid stretched to -1..1 and centred on 0, so a layer of tanh units passes on values that average near zero, and its slope reaches 1 at the centre. It still saturates: at 2.5 the output is 0.9866 and the slope only 0.0266. The red band marks every input where the slope is under 0.1, beyond about 1.8 on either side, and a unit whose inputs live there barely learns. The spec tanh_activation.t27 computes the curve, the slope and its own batch, tanh of 0, 1 and 2: 0, 0.762 and 0.964.",
+ "GELU weighs x by the chance that a standard normal draw is below it: GELU(x) = x Phi(x). A large positive input passes, a large negative one vanishes, and in between the curve dips to about -0.17 near x = -0.75 before it rises. BERT and GPT use it in their feed-forward layers. The exact form needs erf, which the spec computes with the Abramowitz-Stegun fit, good to 1.5e-7, and it also computes the tanh shortcut: at -0.75 they give -0.16997 and -0.17004. The spec gelu_activation.t27 computes both curves and their slopes.",
+ "Swish is x times sigmoid(beta x), a unit that gates itself. At beta 1 it is called SiLU, and its lowest point is -0.2785 near x = -1.28. Raise beta and the gate sharpens: at beta 8 the curve is ReLU with rounded corners; at beta 0.5 it is a gentle slope through zero. One parameter moves the unit between a smooth curve and a switch, and a network can learn it. The spec silu_swish_activation.t27 computes the curve and its slope for any beta, and its own batch at -2, 0 and 2: -0.238, 0 and 1.762.",
+ "ELU keeps the positive side of ReLU and replaces the cliff on the negative side with alpha (e^x - 1), a curve that flattens toward -alpha. With alpha 1, x = -3 gives -0.9502, already near the floor of -1, and the slope there is 0.0498: small, but not 0, so the unit can still move. Because the negative outputs are not thrown away, a layer's outputs average closer to 0. The spec elu_activation.t27 computes the curve, the slope and the floor, and its own batch at -1, 0 and 2: -0.632, 0 and 2.",
+ "A classifier ends with one score per class, and softmax turns the scores into probabilities: exp of each score over the sum of all of them. Scores 1, 2 and 3 become 0.090, 0.245 and 0.665. Dividing the scores by a temperature first changes how sure it is: at 0.5 the top score takes 0.867, at 5 the three are 0.27, 0.33 and 0.40. A language model picks its next token from exactly this, and the temperature setting of a chat model is this T. The spec softmax.t27 subtracts the largest score first, so exp never overflows.",
+ "A network cannot multiply a word, so each token is an index into a table with one row of numbers per token, and looking a token up copies its row. In the spec the table has 4 tokens of width 2 and row v holds (10 v, 10 v + 1), so token 2 reads (20, 21). Learning sends the gradient back only to the rows that were read, and a token read twice collects both: gradient rows (1, 2) and (3, 4) on token 2 add up to (4, 6). The spec embedding_layer.t27 computes the lookup and the gradient.",
+ "While training, dropout switches off a share p of the units at random, so the rest cannot lean on any one of them. The units that survive must then be scaled, or the next layer would see the signal at another size in training than at inference. Inverted dropout scales them by 1 / (1 - p) while training: with p = 0.5, input 1, 2, 3, 4 becomes 2, 0, 6, 0, and inference is a plain pass. Classic dropout keeps 1, 0, 3, 0 and scales by 0.5 at inference instead. The spec dropout_layer.t27 uses a fixed mask, keep, drop, keep, drop, so the numbers are the same every time.",
+ "A picture is a grid of numbers, and networks read it in small patches so that the same detector works anywhere in it. Here a 2 x 2 window steps by 2 over a 4 x 4 image whose cell i holds i, and each output is the average of its patch: 2.5, 4.5, 10.5 and 12.5. In the backward pass each output's gradient is split equally over its 4 cells. This is average pooling: a convolution would weight the 4 cells with a learned kernel, and its spec, t27#10612, is not merged yet. The spec avgpool2d_layer.t27 computes both passes.",
+ "Max pooling keeps the largest value of each window and drops the rest, so the network cares less about exactly where a feature sits. On the spec's 4 x 4 grid, 2 x 2 windows with stride 2 keep 4, 8, 9 and -2, taken from cells 5, 6, 13 and 11. Overlapping 3 x 3 windows with stride 1 keep 8, 8, 9 and 9. In the backward pass each window's whole gradient goes to its maximum: gradients 1, 2, 3 and 4 land on cells 5, 6, 13 and 11, and every other cell gets 0. The spec maxpool2d_layer.t27 computes the outputs, their positions and the gradient.",
+ "Features on different scales pull a network around: in the spec's batch one feature is ten times another. Batch normalisation subtracts each feature's batch mean and divides by its spread, so the features 1, 2, 3, 4 and 10, 20, 30, 40 both come out as -1.34, -0.45, 0.45 and 1.34, and -1, -1, 1, 1 comes out as -1 and 1. A learned gamma and beta then scale and shift it back where that helps. At inference the layer uses running averages instead of the batch: with mean 0, variance 1, gamma 1 and beta 0.25, the inputs 1, 2 and 3 give 1.25, 2.25 and 3.25. The spec batchnorm_layer.t27 computes both.",
+ "A loss turns all the misses of a model into one number to push down. Mean squared error averages the squared misses: predictions 1.5, 2 and 2 against targets 1, 2 and 3 cost (0.25 + 0 + 1) / 3 = 0.4167. Squaring makes a miss of 2 cost 4 times a miss of 1, so MSE chases large errors. Its gradient for each prediction is 2 (p - t) / n, here 0.333, 0 and -0.667: each prediction is pushed back toward its target. As a function of one prediction the loss is a parabola, a bowl with one bottom. The spec mse_loss.t27 computes the loss and the gradient.",
+ "A classifier outputs a probability, and cross-entropy charges -log of the probability it gave to the true answer. Predictions 0.8, 0.3 and 0.9 against answers 1, 0 and 1 cost 0.2284 on average. Being sure and right is nearly free: 0.99, 0.01 and 0.99 cost 0.0101. Being sure and wrong is expensive: 0.01, 0.99 and 0.01 cost 4.6052, and toward 0 the cost grows without bound, which is why the spec clips p to [1e-7, 1 - 1e-7]. Its gradient is largest exactly where the model is sure and wrong. The spec binary_crossentropy_loss.t27 computes the loss and the gradient, from probabilities or from logits.",
+ "Squared error lets one outlier dominate: an error of 3 costs 4.5 as half its square, as much as 36 errors of 0.5. Huber is half the square while |e| is within delta and then grows only linearly, delta |e| - delta^2 / 2, so with delta 1 that error costs 2.5. The two pieces meet with the same value and slope, and the gradient never grows past delta: a far-off point pulls on the model with a bounded force. For the spec's pair of errors 0.5 and 3 the mean is 1.3125, against 2.3125 for half the square. The spec huber_loss.t27 computes both.",
+ "Training is walking downhill on the loss. Gradient descent moves a weight by -lr times its gradient each step; in the spec the loss is (x - 3)^2 from x = 0, and lr 0.1 reaches 2.99996 in 50 steps. Momentum adds a share of the last move, which speeds up a long slope and overshoots a sharp bottom: at 0.9 the weight shoots past 3 to about 5.13 and rings down to 3.09 after 50 steps. Nesterov takes the gradient one step ahead and swings less, to about 4.25. Too large a rate never settles: at lr 1 the weight bounces between 0 and 6 for good. The spec sgd.t27 computes every step.",
+ "The right step size depends on the size of the gradient, and different weights see gradients a million times apart. RMSProp keeps a running mean of the squared gradient, the cache, and divides each step by its root, so the step has no units: with a constant gradient of 0.001, 0.5 or 1000 the weight moves by exactly the same 0.0316 on step 1 and walks the same path, reaching -0.0914 after 100 steps at lr 0.01. The decay sets how slowly the cache fills: at 0.99 the same 100 steps end at -1.0261. The spec rmsprop.t27 computes the weight and the cache.",
+ "Adam combines the two ideas: m, a running mean of the gradient, as in momentum, and v, a running mean of its square, as in RMSProp; the step is lr m / sqrt(v). Both start at 0 and are too small early on, so Adam divides them by 1 - beta^t, and on step 1 that makes the move exactly lr times the sign of the gradient: gradients 2, -40 and 0.5 all move their weights by 0.001. On the spec's bowl (x - 3)^2 at lr 0.1, Adam rises from 0, overshoots to about 3.18 and settles at 3.00005 by step 200. The spec adam.t27 computes every step.",
+ "To learn, every weight needs to know how the loss changes when it moves. Backpropagation answers one layer at a time: given g, how much the loss wants each output to move, the weight from input i to output o gets g_o x_i, and input i gets the sum over o of W[o][i] g_o, which it hands on to the layer below. With inputs (1, 2, 3) and g = (1, 0.5) the weight gradients are 1, 2, 3 and 0.5, 1, 1.5, and the input gradients 0.5, 2 and 3.5. The layer is linear, so moving x0 by 1 moves the loss g . y from 15.25 to 15.75, by its gradient 0.5. The spec dense_layer.t27 computes all three.",
+ "XOR is 1 when exactly one of two inputs is 1. No straight line separates the cases that give 1 from those that give 0, so a single neuron cannot learn it. A hidden layer bends the space: the spec trains a 2-8-1 network of ReLUs by backpropagation, lr 0.05, from fixed initial weights. Its loss falls from 0.8896 in epoch 1 to 1.8e-13 after 2000 epochs, and its four answers come out within 4e-7 of 0, 1, 1 and 0. The widget runs that training in your browser, from scratch for every point of the curve. The spec mlp.t27 holds the network and the test that it learns.",
+ "Training with very large batches needs a large learning rate, and one rate can be too large for a small layer and too small for a big one. LAMB takes an Adam-style update for each layer and rescales it by the trust ratio, the layer's weight norm, capped at 10, over the update's norm, so every layer moves by the same fraction of itself. In the spec, the weights (1, 2) times a scale take a step whose norm is 0.001 times theirs: 0.00447 at scale 2, and once the weight norm passes 10, near scale 4.47, the step stays at 0.01. The spec lamb.t27 computes the update for one layer and for two at once.",
+ "A sentence comes one word at a time, and a recurrent cell carries a state h from one step to the next: h' = tanh(W_x x + W_h h + b). In the spec's tiny cell W_x is 1 and W_h 0.5, so a single pulse gives 0.7616, 0.3634 and 0.1797: the memory of it roughly halves each step. A second pulse at step 3 lifts h to 0.7968, a little above where the single pulse began, because the old memory is still there. After 8 steps only 0.0466 is left. That fading is why plain cells forget long contexts. The spec rnn_cell.t27 computes each step.",
+ "An LSTM cell keeps a separate memory c and three gates, each a sigmoid between 0 and 1: forget scales the old memory, input lets in new content g, and output decides how much of the memory to show. c' = f c + i g and h' = o tanh(c'). In the spec's fixture every weight is 0, so each gate is the sigmoid of its bias: with the forget gate at 0.9975 and the input gate at 0.0025, a memory of 1 comes out as 0.9975, and the output is 0.7587. Because c passes through with only a multiply by f, its gradient survives many steps too. The spec lstm_single.t27 computes the cell.",
+ "Attention lets each position decide which other positions to read. Every query is scored against every key, q . k / sqrt(d), and a softmax turns each row of scores into weights that sum to 1; the output is the values averaged by those weights. Two heads do it on the two halves of the vector. With a causal mask a position cannot read the future: in head 0, position 0 reads only itself, position 1 splits 0.452 and 0.548, and position 2 spreads 0.230, 0.321 and 0.449. Without the mask every row spreads over all three keys, from 0.21 to 0.47. The spec attention_mechanism.t27 computes every weight on its own fixture.",
+ "Attention treats its input as a set, so the order of the words has to be added to them. The sinusoidal encoding gives each position d numbers: the sin and cos of the position times a frequency that falls by 10000^(2/d) from pair to pair. With d = 8 the four pairs turn by 1, 0.1, 0.01 and 0.001 radians per position, like the hands of four clocks: position 5 reads -0.959, 0.284 in its fast pair and 0.005, 1.000 in its slow one. Each pair has length 1, and a shift by k positions turns every pair by its own fixed angle. The spec positional_encoding.t27 computes every value.",
+ "A transformer layer is two sublayers, each wrapped in a residual add and a layer norm: self-attention mixes the positions, then a feed-forward net of width 8 works on each position alone. In the post-norm order of the original paper the block ends with a norm, so every output row has mean 0 and variance 1.000. Pre-norm puts the norms in front of each sublayer and ends on a residual add, so its rows come out at variances 2.73, 3.33 and 0.96; deep stacks train more easily that way, with one norm added at the very top. The spec encoder_block.t27 runs one block on 3 positions, d_model 4 and 2 heads.",
+ "An agent learns from reward by making good actions more likely. PPO measures a change of policy by the ratio r of the new probability of an action to the old one and multiplies it by the advantage A, how much better the action did than expected. Then it clips: once r has moved more than eps the way A rewards, the objective goes flat. At r = 1.5, A = 1 and eps 0.2 the raw r A is 1.5, but PPO counts 1.2, so one batch cannot push the policy far. On the spec's batch, ratios 1, 1.1 and 0.9 with advantages 2, 1.5 and 3, the policy loss is -2.1167. The spec ppo_clip_loss.t27 computes it."
+];
+pub const LESSON_TASKS : [27]str = [
+ "Put x at -2 and read the slope with and without the leak; then find the slope at x = 1.5.",
+ "Find the input where the probability is 0.5; then drag to 6 and read how little slope is left.",
+ "Drag x until the band starts and read the slope there; then compare the slope at 0 and at 2.5.",
+ "Find the lowest point of the curve by its slope; then compare the exact and the tanh form at -3.",
+ "Hold x at -1.28 and raise beta from 1 to 8; then find the beta at which the dip is gone at this scale.",
+ "Set alpha to 0.5 and read the floor; then find how far left x must go before the slope is under 0.01.",
+ "Cool the temperature to 0.1 and read the three probabilities; then make a tie and heat it.",
+ "Read token 2 twice and then tokens 1 and 3; count the rows whose gradient is not 0 each time.",
+ "Compare the outputs of the four modes; then read the gradient of a dropped unit in training.",
+ "Tap each patch and check its average by eye; then switch to the gradient and read what each cell of window 4 receives.",
+ "Switch to 3 x 3 windows and find the cell two outputs share; then turn on the gradient for 2 x 2 windows.",
+ "Compare the two normalised columns; then set the running mean 2 and variance 4 and read the inference outputs.",
+ "Move prediction 2 onto its target and read the loss; then drag along the bowl and find its bottom.",
+ "Drag prediction 0 toward 0.01 and watch the curve; then read the gradient bars at 0.5, 0.5 and 0.5.",
+ "Set delta to 0.5 and drag the error to 3; then find where the two curves part.",
+ "Find the largest learning rate that still settles without momentum; then add momentum 0.9 and count the swings.",
+ "Switch between the three gradients and check that the lines do not move; then raise the decay to 0.99.",
+ "Give the three gradients very different sizes and read the weights after step 1; then find where the path tops out.",
+ "Set g = (1, 0) and read which edges go dark; then set g = (-2, 1) and check the input gradients by hand.",
+ "Drag the epochs back to 100 and read the four answers; then find the epoch where the curve stops falling.",
+ "Read the step at scales 1 and 2 and compare; then drag past the bend and explain why the line goes flat.",
+ "Load one pulse and read how many steps it takes to fall under 0.1; then make the input alternate.",
+ "Load Remember and then Forget and compare the new memory; then write with the input gate open.",
+ "Turn the mask off and on and read row 0; then compare the two heads on row 2.",
+ "Step from position 0 to 1 and watch which clock moves; then jump to 37 and find the pair that still points up.",
+ "Switch between post-norm and pre-norm and read the row variances; then find the position head 1 reads most.",
+ "Make the advantage negative and find where the curve goes flat now; then widen eps to 0.5."
+];
+
+; --- Claims --------------------------------------------------------------------------------
+test the_course_is_three_cubed {
+ assert LESSONS_PER_MODULE == 3;
+ assert MODULE_COUNT == 9;
+ assert LESSON_COUNT == 27;
+ assert MODULE_COUNT * LESSONS_PER_MODULE == LESSON_COUNT;
+ assert LESSON_COUNT == 3 * 3 * 3;
+}
+
+test it_starts_at_one_neuron_and_ends_with_a_reward {
+ assert MODULE_IDS[0] == "one-neuron";
+ assert LESSON_WIDGETS[0] == "nn-relu";
+ assert LESSON_SPECS[0] == "specs/ml/activation/relu_activation.t27";
+ assert MODULE_IDS[8] == "transformer-and-reward";
+ assert LESSON_WIDGETS[26] == "nn-ppo";
+ assert LESSON_SPECS[26] == "specs/ml/rl/ppo_clip_loss.t27";
+}
+
+test activations_come_before_layers {
+ assert MODULE_IDS[1] == "new-bends";
+ assert LESSON_WIDGETS[3] == "nn-gelu";
+ assert MODULE_IDS[2] == "a-layer";
+ assert LESSON_SPECS[6] == "specs/ml/activation/softmax.t27";
+}
+
+test the_picture_lesson_is_pooling_until_conv2d_slides {
+ assert MODULE_IDS[3] == "pictures";
+ assert LESSON_WIDGETS[9] == "nn-conv";
+ assert LESSON_SPECS[9] == "specs/ml/layers/avgpool2d_layer.t27";
+ assert LESSON_WIDGETS[10] == "nn-maxpool";
+}
+
+test losses_then_optimizers_then_learning {
+ assert MODULE_IDS[4] == "the-error";
+ assert MODULE_IDS[5] == "downhill";
+ assert LESSON_WIDGETS[17] == "nn-adam";
+ assert MODULE_IDS[6] == "learning";
+ assert LESSON_WIDGETS[19] == "nn-xor";
+ assert LESSON_SPECS[19] == "specs/ml/pathway/mlp.t27";
+}
+
+test memory_then_attention_then_the_transformer {
+ assert MODULE_IDS[7] == "memory-and-attention";
+ assert LESSON_WIDGETS[23] == "nn-attention";
+ assert LESSON_WIDGETS[25] == "nn-encoder";
+ assert LESSON_SPECS[25] == "specs/ml/transformer/encoder_block.t27";
+}
+
+test the_course_sends_nothing {
+ assert SENDS_NOTHING == true;
+ assert LOCALES[0] == "en";
+ assert LOCALES[1] == "ru";
+}
diff --git a/apps/website/public/learn/neural-networks/card.png b/apps/website/public/learn/neural-networks/card.png
new file mode 100644
index 0000000000..e02c1f1f95
Binary files /dev/null and b/apps/website/public/learn/neural-networks/card.png differ
diff --git a/apps/website/public/learn/neural-networks/index.html b/apps/website/public/learn/neural-networks/index.html
new file mode 100644
index 0000000000..8b35b51a94
--- /dev/null
+++ b/apps/website/public/learn/neural-networks/index.html
@@ -0,0 +1,72 @@
+
+
+
+
+
+
+Neural networks in t27: from one neuron to a transformer block · t27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiРусский
+
+
Course
+
Neural networks in t27: from one neuron to a transformer block
+
The agents of this project think with neural networks, and here their parts are written as t27 specs with tests, specs the compiler checks. Every widget runs the fn bodies of a specs/ml spec compiled to WebAssembly: move a slider and the spec answers.
+
Course 10, 27 lessons, a live widget and a t27 spec each: activations, softmax and embeddings, pooling and batch norm, losses, SGD, RMSProp and Adam, backpropagation and XOR, recurrent cells, attention, a transformer block and PPO. Every number is computed by a compiled spec.
Why networks read a picture in patches, and what a 2 x 2 window averaged over each patch gives.
+
A picture is a grid of numbers, and networks read it in small patches so that the same detector works anywhere in it. Here a 2 x 2 window steps by 2 over a 4 x 4 image whose cell i holds i, and each output is the average of its patch: 2.5, 4.5, 10.5 and 12.5. In the backward pass each output's gradient is split equally over its 4 cells. This is average pooling: a convolution would weight the 4 cells with a learned kernel, and its spec, t27#10612, is not merged yet. The spec avgpool2d_layer.t27 computes both passes.
+
Try it
Tap each patch and check its average by eye; then switch to the gradient and read what each cell of window 4 receives.
Tap a window on the 4 x 4 image: each output is the average of its 2 x 2 patch, two cells apart. Then switch to the gradient and see each output's blame shared out over its patch.
+
specs/ml/layers/avgpool2d_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// 2D average pooling formula | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the kernel and stride were string constants typed `[]u32`, forward only
+// asserted that its input was non-empty, and the tests asserted `true` or
+// `result != undefined`. Average pooling now runs over one square H x H channel
+// with the default 2 x 2 window and stride 2 and no padding, and each test checks
+// numbers.
+
+module Avgpool2d;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_KERNEL : u32 = 2;
+ const DEFAULT_STRIDE : u32 = 2;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PoolConfig = struct {
+ kernel_size : []u32,
+ stride : []u32,
+ padding : []u32,
+ };
+
+ pub const PoolShape = struct {
+ batch : u32,
+ channels : u32,
+ height_in : u32,
+ width_in : u32,
+ height_out : u32,
+ width_out : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Output side for an input side n: (n - k) / stride + 1, and 0 when no window fits
+ fn calc_output_size(height_in: u32) -> u32 {
+ if (height_in < DEFAULT_KERNEL) {
+ return 0;
+ }
+ return (height_in - DEFAULT_KERNEL) / DEFAULT_STRIDE + 1;
+ }
+
+ // output[oy][ox] = mean of the k x k window at (oy * stride, ox * stride) of a side x side
+ // input; returns the number of outputs written
+ fn forward(input: []const f32, side: u32, output: []f32) -> u32 {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var sum : f32 = 0.0;
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ sum = sum + input[(oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx];
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * n + ox] = sum / area;
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ return n * n;
+ }
+
+ // backward: each window's gradient is shared equally by its k * k inputs
+ fn backward(grad_output: []const f32, side: u32, grad_input: []f32) -> void {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ const idx = (oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx;
+ grad_input[idx] = grad_input[idx] + grad_output[oy * n + ox] / area;
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ }
+
+ // 4 x 4 input 0..15, row-major: windows average to 2.5, 4.5, 10.5, 12.5
+ fn pool4_at(index: usize) -> f32 {
+ var input : [16]f32 = [0.0] ** 16;
+ var i : usize = 0;
+ while (i < 16) {
+ input[i] = @as(f32, @floatFromInt(i));
+ i = i + 1;
+ }
+ var output : [4]f32 = [0.0] ** 4;
+ _ = forward(&input, 4, &output);
+ return output[index];
+ }
+
+ // forward over a 2 x 2 input (1, 2, 3, 4) or a 1 x 1 input (5); returns the outputs written
+ fn pool_count(side: u32) -> u32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const one : [1]f32 = [5.0];
+ var output : [1]f32 = [-1.0];
+ if (side == 2) {
+ return forward(&four, 2, &output);
+ }
+ return forward(&one, 1, &output);
+ }
+
+ fn pool2_value() -> f32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [1]f32 = [-1.0];
+ _ = forward(&four, 2, &output);
+ return output[0];
+ }
+
+ fn backward4_at(index: usize) -> f32 {
+ const grad_output : [4]f32 = [4.0, 8.0, 12.0, 16.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, 4, &grad_input);
+ return grad_input[index];
+ }
+
+ fn valid_input() -> u32 {
+ return calc_output_size(4);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test calc_output_size_basic_case
+ given result = calc_output_size(4)
+ then result == 2
+
+ test calc_output_size_valid_input
+ given input = 10
+ when result = calc_output_size(input)
+ then result == 5
+
+ // A 1 x 1 input holds no 2 x 2 window
+ test calc_output_size_small_input
+ given input = 1
+ when result = calc_output_size(input)
+ then result == 0 and calc_output_size(2) == 1
+
+ test calc_output_size_large_input
+ given input = 1000
+ when result = calc_output_size(input)
+ then result == 500 and calc_output_size(1001) == 500
+
+ test forward_basic_case
+ given first = pool4_at(0)
+ when last = pool4_at(3)
+ then first == 2.5 and pool4_at(1) == 4.5 and pool4_at(2) == 10.5 and last == 12.5
+
+ // A 2 x 2 input is one window: the mean of 1, 2, 3, 4
+ test forward_non_empty_input
+ given count = pool_count(2)
+ when value = pool2_value()
+ then count == 1 and value == 2.5
+
+ test forward_single_element
+ given count = pool_count(1)
+ then count == 0
+
+ // Each of a window's 4 inputs gets a quarter of its gradient
+ test backward_spreads_the_gradient
+ given corner = backward4_at(0)
+ when last = backward4_at(15)
+ then corner == 1.0 and backward4_at(5) == 1.0 and backward4_at(2) == 2.0 and last == 4.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // kernel_size[i] >= 1
+ invariant avgpool2d_constraint_0
+ assert DEFAULT_KERNEL >= 1
+
+ // stride[i] >= 1
+ invariant avgpool2d_constraint_1
+ assert DEFAULT_STRIDE >= 1
+
+ // stride[i] <= kernel_size[i]: windows tile the input without gaps
+ invariant avgpool2d_constraint_2
+ assert DEFAULT_STRIDE <= DEFAULT_KERNEL
+
+ // name: avgpool_formula -- (n - k) / s + 1
+ invariant avgpool2d_constraint_3
+ assert valid_input() == 2
+
+ // "Zeiler & Fergus (2014) - Visualizing and Understanding Convolutional Networks": pooling halves each side
+ invariant avgpool2d_constraint_4
+ assert calc_output_size(8) == 4
+
+ // "He et al. (2016) - Deep Residual Learning for Image Recognition": a 7 x 7 map pools to 3 x 3
+ invariant avgpool2d_constraint_5
+ assert calc_output_size(7) == 3
+
How sines and cosines at falling frequencies give every position its own vector, and why a shift is a turn.
+
Attention treats its input as a set, so the order of the words has to be added to them. The sinusoidal encoding gives each position d numbers: the sin and cos of the position times a frequency that falls by 10000^(2/d) from pair to pair. With d = 8 the four pairs turn by 1, 0.1, 0.01 and 0.001 radians per position, like the hands of four clocks: position 5 reads -0.959, 0.284 in its fast pair and 0.005, 1.000 in its slow one. Each pair has length 1, and a shift by k positions turns every pair by its own fixed angle. The spec positional_encoding.t27 computes every value.
+
Try it
Step from position 0 to 1 and watch which clock moves; then jump to 37 and find the pair that still points up.
Every position gets 8 numbers, sines and cosines at frequencies falling by 10 per pair. Drag the position: the fast clock spins, the slow ones barely move, and the stripes show all 48 rows.
+
specs/ml/transformer/positional_encoding.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Uses sin/cos at different frequencies for each dimension | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward read an undeclared `d_model`, called `math::pow` / `math::sin` /
+// `math::cos`, computed each encoding into a local and dropped it, and grew the
+// frequency with the dimension where Vaswani et al. (2017) shrink it. It now
+// writes the sinusoidal encoding
+// PE(pos, 2i) = sin(pos / 10000^(2i / d)), PE(pos, 2i + 1) = cos(pos / 10000^(2i / d))
+// for each position into a positions x d_model buffer.
+
+module PositionalEnc;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_MAX_LEN : u32 = 2048;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PosEncConfig = struct {
+ d_model : u32,
+ max_len : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // 1 / 10000^(2i / d) = exp(-(2i / d) ln 10000)
+ fn inv_freq(pair: u32, d_model: u32) -> f32 {
+ return exp(-(@as(f32, @floatFromInt(2 * pair)) / @as(f32, @floatFromInt(d_model))) * log(@as(f32, 10000.0)));
+ }
+
+ fn forward(positions: []const u32, config: PosEncConfig, out: []f32) -> void {
+ var p : usize = 0;
+ while (p < positions.len) {
+ const pos : f32 = @as(f32, @floatFromInt(positions[p]));
+ var dim : u32 = 0;
+ while (dim < config.d_model) {
+ const angle = pos * inv_freq(dim / 2, config.d_model);
+ if (dim % 2 == 0) {
+ out[p * config.d_model + dim] = sin(angle);
+ } else {
+ out[p * config.d_model + dim] = cos(angle);
+ }
+ dim = dim + 1;
+ }
+ p = p + 1;
+ }
+ }
+
+ // PE(pos, dim) for d_model 8
+ fn pe(pos: u32, dim: usize) -> f32 {
+ const positions : [1]u32 = [pos];
+ var out : [8]f32 = [9.0] ** 8;
+ forward(&positions, PosEncConfig{ .d_model = 8, .max_len = DEFAULT_MAX_LEN }, &out);
+ return out[dim];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // pos 1, d 8: dims 0/1 use frequency 1, dims 2/3 use 1/10 (10000^(2/8) = 10)
+ test forward_basic_case
+ given s0 = pe(1, 0)
+ when s2 = pe(1, 2)
+ then @abs(s0 - 0.8414710) < 0.000001 and @abs(pe(1, 1) - 0.5403023) < 0.000001 and @abs(s2 - 0.0998334) < 0.000001 and @abs(pe(1, 3) - 0.9950042) < 0.000001
+
+ // Position 0 encodes as (0, 1, 0, 1, ...)
+ test forward_simple
+ given s = pe(0, 6)
+ when c = pe(0, 7)
+ then s == 0.0 and c == 1.0
+
+ // Each sin/cos pair lies on the unit circle
+ test forward_pairs_are_unit_vectors
+ given p = pe(37, 2) * pe(37, 2) + pe(37, 3) * pe(37, 3)
+ then @abs(p - 1.0) < 0.000001
+
+ invariant positional_encoding_max_len_is_positive
+ assert DEFAULT_MAX_LEN > 0
+
How PPO turns a reward into a policy update, and how its clip keeps one update from going too far.
+
An agent learns from reward by making good actions more likely. PPO measures a change of policy by the ratio r of the new probability of an action to the old one and multiplies it by the advantage A, how much better the action did than expected. Then it clips: once r has moved more than eps the way A rewards, the objective goes flat. At r = 1.5, A = 1 and eps 0.2 the raw r A is 1.5, but PPO counts 1.2, so one batch cannot push the policy far. On the spec's batch, ratios 1, 1.1 and 0.9 with advantages 2, 1.5 and 3, the policy loss is -2.1167. The spec ppo_clip_loss.t27 computes it.
+
Try it
Make the advantage negative and find where the curve goes flat now; then widen eps to 0.5.
Drag the probability ratio of a new policy over the old: PPO's objective follows r times the advantage until r moves more than eps the rewarded way, then it goes flat and stops pulling.
+
specs/ml/rl/ppo_clip_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Probability ratio between policies | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written with `let x = if ... {}` and `for i in 0..n`, so it never compiled.
+// Three formulas were wrong, and their tests pinned the wrong values:
+// - compute_ratio returned log pi_new - log pi_old; the ratio is exp of that.
+// - clipped_surrogate returned clip(r) A; PPO's term is min(r A, clip(r) A), so
+// r = 1.1 inside the clip range gives 2.2, not 2.4.
+// - policy_loss returned the surrogate itself, a quantity to MAXIMISE; as a loss
+// added to the value and entropy losses it is minus the mean surrogate.
+// Each test now states the value PPO defines (Schulman et al., 2017).
+
+module PpoClipLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_CLIP_EPSILON : f32 = 0.2;
+ const DEFAULT_ENTROPY_COEF : f32 = 0.01;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PPOClipConfig = struct {
+ clip_epsilon : f32,
+ entropy_coef : f32,
+ value_clip_coef : f32,
+ };
+
+ pub const PPOBatch = struct {
+ states : [][]f32,
+ actions : [][]f32,
+ old_log_probs : []f32,
+ advantages : []f32,
+ returns : []f32,
+ values : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // r = pi_new / pi_old = exp(log pi_new - log pi_old)
+ fn compute_ratio(new_log_prob: f32, old_log_prob: f32) -> f32 {
+ return exp(new_log_prob - old_log_prob);
+ }
+
+ // min(r A, clip(r, 1 - eps, 1 + eps) A)
+ fn clipped_surrogate(ratio: f32, advantage: f32, clip_epsilon: f32) -> f32 {
+ const clipped_ratio = @min(@max(ratio, 1.0 - clip_epsilon), 1.0 + clip_epsilon);
+ return @min(ratio * advantage, clipped_ratio * advantage);
+ }
+
+ // -mean_t min(r_t A_t, clip(r_t) A_t): the loss gradient descent minimises
+ fn policy_loss(ratios: []const f32, advantages: []const f32, clip_epsilon: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < ratios.len) {
+ total = total + clipped_surrogate(ratios[i], advantages[i], clip_epsilon);
+ i = i + 1;
+ }
+ return -total / @as(f32, @floatFromInt(ratios.len));
+ }
+
+ // -coef * mean entropy: lower loss for a more exploratory policy
+ fn entropy_loss(entropies: []const f32, entropy_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < entropies.len) {
+ total = total + entropies[i];
+ i = i + 1;
+ }
+ return -entropy_coef * (total / @as(f32, @floatFromInt(entropies.len)));
+ }
+
+ // mean 0.5 (target - (old + clip(pred - old, -c, c)))^2
+ fn value_loss(predicted_values: []const f32, target_values: []const f32, old_values: []const f32, clip_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predicted_values.len) {
+ const diff = @min(@max(predicted_values[i] - old_values[i], -clip_coef), clip_coef);
+ const err = target_values[i] - (old_values[i] + diff);
+ total = total + 0.5 * err * err;
+ i = i + 1;
+ }
+ return total / @as(f32, @floatFromInt(predicted_values.len));
+ }
+
+ fn policy_loss_of(r0: f32, r1: f32, r2: f32) -> f32 {
+ const ratios : [3]f32 = [r0, r1, r2];
+ const advantages : [3]f32 = [2.0, 1.5, 3.0];
+ return policy_loss(&ratios, &advantages, DEFAULT_CLIP_EPSILON);
+ }
+
+ fn entropy_loss_of() -> f32 {
+ const entropies : [3]f32 = [0.5, 0.6, 0.4];
+ return entropy_loss(&entropies, DEFAULT_ENTROPY_COEF);
+ }
+
+ fn value_loss_of(p0: f32) -> f32 {
+ const predicted : [3]f32 = [p0, 2.0, 1.5];
+ const target : [3]f32 = [1.1, 2.1, 1.6];
+ const old : [3]f32 = [0.9, 1.9, 1.4];
+ return value_loss(&predicted, &target, &old, 0.2);
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_CLIP_EPSILON;
+ }
+
+ // total_loss(policy_loss_value: f32, value_loss_value: f32, entropy_loss_value: f32, value_loss_coef: f32) → f32
+ fn total_loss(policy_loss_value: f32, value_loss_value: f32, entropy_loss_value: f32, value_loss_coef: f32) -> f32 {
+ // Combine all loss components
+ return policy_loss_value + value_loss_coef * value_loss_value + entropy_loss_value;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test compute_ratio_basic_case
+ given
+ new_log_prob = 0.5
+ old_log_prob = 0.3
+ when result = compute_ratio(new_log_prob, old_log_prob)
+ then @abs(result - 1.2214028) < 0.000001 // exp(0.2); the old `== 0.2` was the log ratio
+
+ test clipped_surrogate_basic_case
+ given
+ ratio = 1.1
+ advantage = 2.0
+ clip_epsilon = 0.2
+ when result = clipped_surrogate(ratio, advantage, clip_epsilon)
+ then @abs(result - 1.1 * 2.0) < 0.000001 // inside [0.8, 1.2] the clip does nothing
+
+ test clipped_surrogate_clipping_case
+ given
+ ratio = 1.5 // Above clip threshold of 1.2
+ advantage = 2.0
+ clip_epsilon = 0.2
+ when result = clipped_surrogate(ratio, advantage, clip_epsilon)
+ then @abs(result - 1.2 * 2.0) < 0.000001 and clipped_surrogate(0.5, -1.0, 0.2) == -0.8
+
+ // min(r A, clip(r) A) = (2.0, 1.65, 2.7): the loss is -mean = -2.1166667. The old `> 0` was
+ // the surrogate's sign; a ratio pushed past the clip earns nothing more.
+ test policy_loss_basic_case
+ given result = policy_loss_of(1.0, 1.1, 0.9)
+ when pushed = policy_loss_of(1.0, 1.1, 3.0)
+ then result < 0.0 and @abs(result + 2.1166667) < 0.000001 and @abs(pushed - policy_loss_of(1.0, 1.1, 1.2)) < 0.000001
+
+ test entropy_loss_basic_case
+ given result = entropy_loss_of()
+ then result < 0.0 and @abs(result + 0.005) < 0.0000001
+
+ test value_loss_basic_case
+ given result = value_loss_of(1.0)
+ when clipped = value_loss_of(5.0)
+ then result > 0.0 and @abs(result - 0.005) < 0.0000001 and @abs(clipped - value_loss_of(1.1)) < 0.0000001
+
+ test total_loss_basic_case
+ given
+ policy_loss_value = 1.0
+ value_loss_value = 0.5
+ entropy_loss_value = -0.01
+ value_loss_coef = 0.5
+ when result = total_loss(policy_loss_value, value_loss_value, entropy_loss_value, value_loss_coef)
+ then @abs(result - (1.0 + 0.5 * 0.5 + (-0.01))) < 0.0000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // 0 < clip_epsilon <= 1 (typically 0.1-0.3)
+ invariant ppo_clip_loss_constraint_0
+ assert valid_input() > 0.0 and valid_input() <= 1.0
+
+ // entropy_coef >= 0
+ invariant ppo_clip_loss_constraint_1
+ assert DEFAULT_ENTROPY_COEF >= 0.0
+
+ // name: ppo_objective -- for A > 0 the surrogate stops growing at r = 1 + eps
+ invariant ppo_clip_loss_constraint_2
+ assert clipped_surrogate(5.0, 1.0, 0.2) == 1.2
+
+ // name: probability_ratio -- equal log-probabilities give r = 1
+ invariant ppo_clip_loss_constraint_3
+ assert compute_ratio(0.3, 0.3) == 1.0
+
+ // "Schulman et al. (2017) - Proximal Policy Optimization Algorithms, arXiv:1707.06347":
+ // for A < 0 the surrogate stops falling at r = 1 - eps
+ invariant ppo_clip_loss_constraint_4
+ assert clipped_surrogate(0.1, -1.0, 0.2) == -0.8
+
What ReLU does to a positive and to a negative input, and why a slope of 0 stops a unit from learning.
+
A neuron adds up its weighted inputs and passes the sum through an activation. ReLU is the plainest one, max(0, x): a positive sum goes through unchanged and a negative one becomes 0. Its slope is 1 or 0, and a slope of 0 is a unit that has stopped learning, because no gradient flows back through it. A leaky slope fixes that: with a slope of 0.1, x = -2 gives -0.2 and passes back a gradient of 0.1. The spec relu_activation.t27 computes both curves the widget draws.
+
Try it
Put x at -2 and read the slope with and without the leak; then find the slope at x = 1.5.
Drag along the curve and watch ReLU pass every positive input and zero every negative one; give it a leaky slope and a little of the negative side gets through.
+
specs/ml/activation/relu_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// ReLU produces sparse gradients | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the tests called forward and backward with one or two of their three or
+// four arguments, two asserted only `result != undefined`, and the invariants
+// asserted `true`. Each test now runs the functions on buffers and checks values.
+
+module Relu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const ZERO : f32 = 0.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const ReLUConfig = struct {
+ negative_slope : f32,
+ inplace : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(input: []const f32, output: []f32, config: ReLUConfig) → void
+ fn forward(input: []const f32, output: []f32, config: ReLUConfig) -> void {
+ // Values at or below ZERO take the negative branch, values above ZERO pass through
+ // For negative_slope = 0.0, this gives max(0, x) behavior
+ var i : usize = 0;
+ while (i < input.len) : (i += 1) {
+ if (input[i] > ZERO) {
+ output[i] = input[i]; // Positive branch: identity
+ } else {
+ output[i] = input[i] * config.negative_slope; // Negative branch: scaled
+ }
+ }
+ }
+
+ // backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) → void
+ fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) -> void {
+ // The gradient gate is on the forward input, not grad_output - this makes gradients sparse
+ var i : usize = 0;
+ while (i < input.len) : (i += 1) {
+ if (input[i] > ZERO) {
+ grad_input[i] = grad_output[i]; // Positive branch: identity gradient
+ } else {
+ grad_input[i] = grad_output[i] * config.negative_slope; // Negative branch: scaled gradient
+ }
+ }
+ }
+
+ // Note: config.inplace is declared but currently unused by these functions.
+ // The buffers are separate parameters, so the aliasing case is not handled.
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ fn cfg(slope: f32) -> ReLUConfig {
+ return ReLUConfig{ .negative_slope = slope, .inplace = false };
+ }
+
+ // output[index] of forward over (a, b, c)
+ fn forward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const input : [3]f32 = [a, b, c];
+ var output : [3]f32 = [9.0] ** 3;
+ forward(&input, &output, cfg(slope));
+ return output[index];
+ }
+
+ // grad_input[index] of backward over (a, b, c), every grad_output 1
+ fn backward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [a, b, c];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &grad_input, cfg(slope));
+ return grad_input[index];
+ }
+
+ // The shared fixture: one negative, zero, one positive.
+ fn valid_input() -> f32 {
+ return -2.0;
+ }
+
+ // max(0, x) on (-2, 0, 2)
+ test "forward_basic_case"
+ given negative = forward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = forward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and forward_at(-2.0, 0.0, 2.0, 0.0, 1) == 0.0 and positive == 2.0
+
+ test "forward_positive_identity"
+ given slope = 0.0
+ when first = forward_at(1.0, 2.0, 3.0, slope, 0)
+ then first == 1.0 and forward_at(1.0, 2.0, 3.0, slope, 1) == 2.0 and forward_at(1.0, 2.0, 3.0, slope, 2) == 3.0
+
+ test "forward_negative_scaling"
+ given slope = 0.5
+ when first = forward_at(-1.0, -2.0, -3.0, slope, 0)
+ then first == -0.5 and forward_at(-1.0, -2.0, -3.0, slope, 1) == -1.0 and forward_at(-1.0, -2.0, -3.0, slope, 2) == -1.5
+
+ test "forward_zero_negative_slope"
+ given slope = 0.0
+ when first = forward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 1) == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+
+ // grad_output * negative_slope for input <= ZERO (0 included), grad_output above it
+ test "backward_input_gate"
+ given slope = 0.5
+ when first = backward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 1) == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+
+ // Plain ReLU: the gradient is 0 left of zero and passes through right of it
+ test "backward_basic_case"
+ given negative = backward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = backward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and positive == 1.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.len == output.len: forward writes every element of its buffer (none keeps the 9.0 fill)
+ invariant relu_constraint_0
+ assert forward_at(valid_input(), 0.0, 1.0, 0.0, 0) != 9.0 and forward_at(valid_input(), 0.0, 1.0, 0.0, 2) != 9.0
+
+ // A finite input gives a finite output: ReLU never exceeds its input's magnitude
+ invariant relu_constraint_1
+ assert @abs(forward_at(valid_input(), 0.0, 1.0, 0.5, 0)) <= @abs(valid_input())
+
How RMSProp divides each step by a running root mean square, so the step no longer depends on the gradient's size.
+
The right step size depends on the size of the gradient, and different weights see gradients a million times apart. RMSProp keeps a running mean of the squared gradient, the cache, and divides each step by its root, so the step has no units: with a constant gradient of 0.001, 0.5 or 1000 the weight moves by exactly the same 0.0316 on step 1 and walks the same path, reaching -0.0914 after 100 steps at lr 0.01. The decay sets how slowly the cache fills: at 0.99 the same 100 steps end at -1.0261. The spec rmsprop.t27 computes the weight and the cache.
+
Try it
Switch between the three gradients and check that the lines do not move; then raise the decay to 0.99.
Feed one weight a constant gradient of 0.001, 0.5 or 1000: RMSProp divides by the root of its running mean square, so all three walk the same path. Change the decay and the path bends.
+
specs/ml/optimizer/rmsprop.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// RMSprop optimizer with moving average of squared gradients | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/rmsprop.tri. The port kept only
+// `grad`, squared grad[0], and threw it away; the test read an undefined
+// default_input().
+
+module Rmsprop;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_DECAY : f32 = 0.9;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const RMSpropConfig = struct {
+ learning_rate : f32,
+ decay : f32,
+ epsilon : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // step(grad, param, cache, config) -> void, in place:
+ // cache = decay * cache + (1 - decay) * grad^2
+ // param -= lr * grad / (sqrt(cache) + epsilon)
+ // The .tri formula reads lr, decay and epsilon but its parameter list
+ // omits them; they come from config here.
+ fn step(grad: []const f32, param: []f32, cache: []f32, config: RMSpropConfig) -> void {
+ var i : usize = 0;
+ while (i < grad.len) {
+ const g = grad[i];
+ cache[i] = config.decay * cache[i] + (1.0 - config.decay) * g * g;
+ param[i] = param[i] - config.learning_rate * g / (@sqrt(cache[i]) + config.epsilon);
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ // run `steps` steps of constant gradient g on one parameter from 1.0
+ fn param_after(g: f32, steps: usize, decay: f32) -> f32 {
+ const grad : [1]f32 = [g];
+ var param : [1]f32 = [1.0];
+ var cache : [1]f32 = [0.0];
+ var k : usize = 0;
+ while (k < steps) {
+ step(&grad, ¶m, &cache, RMSpropConfig{ .learning_rate = 0.01, .decay = decay, .epsilon = 0.00000001 });
+ k = k + 1;
+ }
+ return param[0];
+ }
+
+ fn cache_after(g: f32, steps: usize) -> f32 {
+ const grad : [1]f32 = [g];
+ var param : [1]f32 = [1.0];
+ var cache : [1]f32 = [0.0];
+ var k : usize = 0;
+ while (k < steps) {
+ step(&grad, ¶m, &cache, RMSpropConfig{ .learning_rate = 0.01, .decay = DEFAULT_DECAY, .epsilon = 0.00000001 });
+ k = k + 1;
+ }
+ return cache[0];
+ }
+
+ // first step from cache 0: cache = 0.1 g^2, so the move is lr / sqrt(0.1)
+ // = 0.0316228 whatever the size of g
+ test first_step_is_scale_free
+ given small = param_after(0.001, 1, DEFAULT_DECAY)
+ when large = param_after(1000.0, 1, DEFAULT_DECAY)
+ then @abs(small - (1.0 - 0.0316228)) < 0.00001 and @abs(large - small) < 0.00001
+
+ // moving average of a constant: cache_n = (1 - 0.9^n) g^2
+ test cache_is_moving_average
+ given result = cache_after(2.0, 3)
+ when expected = (1.0 - 0.729) * 4.0
+ then @abs(result - expected) < 0.00001
+
+ test moves_against_the_gradient
+ given up = param_after(0.5, 2, DEFAULT_DECAY)
+ when down = param_after(-0.5, 2, DEFAULT_DECAY)
+ then up < 1.0 and down > 1.0 and @abs((1.0 - up) - (down - 1.0)) < 0.00001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant rmsprop_decay_in_unit_interval
+ assert DEFAULT_DECAY > 0.0 and DEFAULT_DECAY < 1.0
+
How softmax turns scores into probabilities that sum to one, and what the temperature does to them.
+
A classifier ends with one score per class, and softmax turns the scores into probabilities: exp of each score over the sum of all of them. Scores 1, 2 and 3 become 0.090, 0.245 and 0.665. Dividing the scores by a temperature first changes how sure it is: at 0.5 the top score takes 0.867, at 5 the three are 0.27, 0.33 and 0.40. A language model picks its next token from exactly this, and the temperature setting of a chat model is this T. The spec softmax.t27 subtracts the largest score first, so exp never overflows.
+
Try it
Cool the temperature to 0.1 and read the three probabilities; then make a tie and heat it.
Move three scores and the temperature: softmax turns them into probabilities that sum to one. Cool it and the top score takes almost everything; heat it and the choice evens out.
+
specs/ml/activation/softmax.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Control sharpness of distribution | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written with `range` and `math::exp`, which no spec declares, so no test
+// ever ran. backward was also wrong: it used only the Jacobian's diagonal,
+// p_i (1 - p_i), and dropped the -p_i p_j terms. It is now the full product
+// grad_input_i = p_i (g_i - sum_j g_j p_j) / temperature.
+
+module Softmax;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const ZERO : f32 = 0.0;
+ const ONE : f32 = 1.0;
+ const MIN_TEMP : f32 = 0.001;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SoftmaxConfig = struct {
+ temperature : f32,
+ axis : u32,
+ stable : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // output_i = exp((x_i - max) / T) / sum_j exp((x_j - max) / T); subtracting the max keeps exp finite
+ fn forward(logits: []const f32, output: []f32, config: SoftmaxConfig) -> void {
+ var max_val : f32 = logits[0];
+ var i : usize = 0;
+ while (i < logits.len) {
+ if (logits[i] > max_val) {
+ max_val = logits[i];
+ }
+ i = i + 1;
+ }
+ var sum : f32 = ZERO;
+ i = 0;
+ while (i < logits.len) {
+ output[i] = exp((logits[i] - max_val) / config.temperature);
+ sum = sum + output[i];
+ i = i + 1;
+ }
+ i = 0;
+ while (i < logits.len) {
+ output[i] = output[i] / sum;
+ i = i + 1;
+ }
+ }
+
+ // Vector-Jacobian product: grad_input_i = p_i (g_i - sum_j g_j p_j) / T
+ fn backward(grad_output: []const f32, probabilities: []const f32, grad_input: []f32, config: SoftmaxConfig) -> void {
+ var dot : f32 = ZERO;
+ var i : usize = 0;
+ while (i < grad_output.len) {
+ dot = dot + grad_output[i] * probabilities[i];
+ i = i + 1;
+ }
+ i = 0;
+ while (i < grad_output.len) {
+ grad_input[i] = probabilities[i] * (grad_output[i] - dot) / config.temperature;
+ i = i + 1;
+ }
+ }
+
+ fn cfg(temperature: f32) -> SoftmaxConfig {
+ return SoftmaxConfig{ .temperature = temperature, .axis = 0, .stable = true };
+ }
+
+ // output[index] of softmax over (a, b, c)
+ fn softmax_at(a: f32, b: f32, c: f32, temperature: f32, index: usize) -> f32 {
+ const logits : [3]f32 = [a, b, c];
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&logits, &output, cfg(temperature));
+ return output[index];
+ }
+
+ // grad_input[index] for grad_output (0.1, 0.2, 0.3) at probabilities (0.2, 0.3, 0.5)
+ fn backward_at(index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ const probabilities : [3]f32 = [0.2, 0.3, 0.5];
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ // L = sum_i w_i softmax(x)_i with w = (0.1, 0.2, 0.3): the loss whose gradient backward computes
+ fn weighted(a: f32, b: f32, c: f32) -> f32 {
+ return 0.1 * softmax_at(a, b, c, 1.0, 0) + 0.2 * softmax_at(a, b, c, 1.0, 1) + 0.3 * softmax_at(a, b, c, 1.0, 2);
+ }
+
+ fn grad_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ var probabilities : [3]f32 = [0.0] ** 3;
+ const logits : [3]f32 = [a, b, c];
+ forward(&logits, &probabilities, cfg(1.0));
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 3.0;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // softmax(1, 2, 3) = (0.0900306, 0.2447285, 0.6652410)
+ test "forward_basic_case"
+ given p0 = softmax_at(1.0, 2.0, 3.0, 1.0, 0)
+ when p2 = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ then p0 > 0.0 and softmax_at(1.0, 2.0, 3.0, 1.0, 1) > 0.0 and p2 > 0.0 and @abs(p0 + softmax_at(1.0, 2.0, 3.0, 1.0, 1) + p2 - 1.0) < 0.0001 and @abs(p0 - 0.0900306) < 0.000001 and @abs(p2 - 0.6652410) < 0.000001
+
+ // Subtracting the max: logits of 1000 do not overflow exp
+ test "forward_large_logits_stay_finite"
+ given p = softmax_at(1000.0, 1000.0, 1000.0, 1.0, 1)
+ then @abs(p - 1.0 / 3.0) < 0.000001
+
+ // A higher temperature flattens the distribution toward uniform
+ test "forward_temperature_flattens"
+ given sharp = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ when flat = softmax_at(1.0, 2.0, 3.0, 10.0, 2)
+ then flat < sharp and flat > 1.0 / 3.0
+
+ // dot = 0.23: grad = (0.2 (0.1 - 0.23), 0.3 (0.2 - 0.23), 0.5 (0.3 - 0.23)) = (-0.026, -0.009, 0.035)
+ test "backward_basic_case"
+ given g0 = backward_at(0)
+ when g2 = backward_at(2)
+ then g0 != 0.0 and backward_at(1) != 0.0 and g2 != 0.0 and @abs(g0 - (-0.026)) < 0.000001 and @abs(backward_at(1) - (-0.009)) < 0.000001 and @abs(g2 - 0.035) < 0.000001
+
+ // The gradient of a function of probabilities that sum to 1 sums to 0
+ test "backward_sums_to_zero"
+ given total = backward_at(0) + backward_at(1) + backward_at(2)
+ then @abs(total) < 0.0000001
+
+ test "backward_matches_finite_difference"
+ given slope = (weighted(1.0, 2.0 + 0.001, 3.0) - weighted(1.0, 2.0 - 0.001, 3.0)) / 0.002
+ when result = grad_at(1.0, 2.0, 3.0, 1)
+ then @abs(result - slope) < 0.0005
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // logits.size() == output.size(): the first and last element are both probabilities
+ invariant softmax_constraint_0
+ assert softmax_at(1.0, 2.0, valid_input(), 1.0, 0) >= 0.0 and softmax_at(1.0, 2.0, valid_input(), 1.0, 2) >= 0.0
+
+ // config.temperature >= MIN_TEMP
+ invariant softmax_constraint_1
+ assert cfg(1.0).temperature >= MIN_TEMP
+
+ // sum(output) = 1.0 within floating precision
+ invariant softmax_constraint_2
+ assert @abs(softmax_at(1.0, 2.0, valid_input(), 1.0, 0) + softmax_at(1.0, 2.0, valid_input(), 1.0, 1) + softmax_at(1.0, 2.0, valid_input(), 1.0, 2) - 1.0) < 0.0001
+
How x times sigmoid(beta x) gates itself, and how one parameter moves it from a smooth curve to ReLU.
+
Swish is x times sigmoid(beta x), a unit that gates itself. At beta 1 it is called SiLU, and its lowest point is -0.2785 near x = -1.28. Raise beta and the gate sharpens: at beta 8 the curve is ReLU with rounded corners; at beta 0.5 it is a gentle slope through zero. One parameter moves the unit between a smooth curve and a switch, and a network can learn it. The spec silu_swish_activation.t27 computes the curve and its slope for any beta, and its own batch at -2, 0 and 2: -0.238, 0 and 1.762.
+
Try it
Hold x at -1.28 and raise beta from 1 to 8; then find the beta at which the dip is gone at this scale.
Drag along x times sigmoid(beta x). At beta 1 it is SiLU, with its dip of -0.28; turn beta up and the curve closes on ReLU, turn it down and it flattens toward x / 2.
+
specs/ml/activation/silu_swish_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// SiLU derivative | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward_batch returned its input unchanged, beta was never read, and the
+// invariants asserted `true` over an undeclared `valid_input`. SiLU is now
+// x * sigmoid(beta * x) (Swish; beta = 1 is SiLU) on scalars and in place on slices.
+
+module SiluSwish;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_BETA : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SiLUConfig = struct {
+ beta : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // sigmoid(z) = 1 / (1 + exp(-z))
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Swish(x) = x * sigmoid(beta * x); beta = 1 is SiLU
+ fn forward(x: f32, beta: f32) -> f32 {
+ return x * sigmoid(beta * x);
+ }
+
+ // forward on each element IN PLACE; returns the same slice
+ fn forward_batch(input: []f32, beta: f32) -> []f32 {
+ var i : usize = 0;
+ while (i < input.len) {
+ input[i] = forward(input[i], beta);
+ i = i + 1;
+ }
+ return input;
+ }
+
+ // d/dx x sigmoid(beta x) = s + beta x s (1 - s), s = sigmoid(beta x)
+ fn derivative(x: f32, beta: f32) -> f32 {
+ const s = sigmoid(beta * x);
+ return s + beta * x * s * (1.0 - s);
+ }
+
+ fn batch_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ var input : [3]f32 = [a, b, c];
+ const output = forward_batch(&input, DEFAULT_BETA);
+ return output[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test "forward_basic_case"
+ given input = 0.0, beta = 1.0
+ when result = forward(input, beta)
+ then result == 0.0 // SiLU(0) = 0 * sigmoid(0) = 0
+
+ test "forward_positive_case"
+ given input = 1.0, beta = 1.0
+ when result = forward(input, beta)
+ then result > 0.0 // SiLU(1) = 1 * sigmoid(1) > 0
+
+ test "forward_negative_case"
+ given input = -1.0, beta = 1.0
+ when result = forward(input, beta)
+ then result < 0.0 // SiLU(-1) = -1 * sigmoid(-1) < 0
+
+ test "forward_batch_basic_case"
+ given beta = 1.0
+ when result = batch_at(1.0, 2.0, 3.0, 2)
+ then @abs(result - forward(3.0, beta)) < 0.0000001 and @abs(batch_at(1.0, 2.0, 3.0, 0) - 0.7310586) < 0.000001
+
+ test "forward_batch_zero_case"
+ given beta = 1.0
+ when result = batch_at(0.0, 0.0, 0.0, 0)
+ then result == 0.0 and batch_at(0.0, 0.0, 0.0, 1) == 0.0 and batch_at(0.0, 0.0, 0.0, 2) == 0.0 // SiLU(0) = 0
+
+ test "derivative_basic_case"
+ given input = 0.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU'(0) = sigmoid(0) + 0 * sigmoid(0) * (1 - sigmoid(0)) = sigmoid(0) > 0
+
+ test "derivative_positive_case"
+ given input = 1.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU derivative should be positive for positive inputs
+
+ test "derivative_negative_case"
+ given input = -1.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU derivative should be positive for negative inputs too
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // beta > 0
+ invariant silu_swish_constraint_0
+ assert DEFAULT_BETA > 0.0
+
+ // name: silu_formula -- x * sigmoid(x), so SiLU(x) - SiLU(-x) = x
+ invariant silu_swish_constraint_1
+ assert @abs(forward(valid_input(), DEFAULT_BETA) - forward(-valid_input(), DEFAULT_BETA) - valid_input()) < 0.000001
+
+ // name: derivative -- sigmoid(0) = 0.5 at the origin
+ invariant silu_swish_constraint_2
+ assert derivative(0.0, DEFAULT_BETA) == 0.5
+
+ // "Elfwing et al. (2017) - Sigmoid-Weighted Linear Units for Neural Network Function Approximation":
+ // SiLU dips below zero left of the origin
+ invariant silu_swish_constraint_3
+ assert forward(-valid_input(), DEFAULT_BETA) < 0.0
+
+ // "Ramachandran et al. (2017) - Searching for Activation Functions (Swish)": a large beta
+ // approaches ReLU
+ invariant silu_swish_constraint_4
+ assert @abs(forward(valid_input(), 50.0) - valid_input()) < 0.000001
+
Why tanh is centred on zero, and where its slope falls so low that a unit stops learning.
+
tanh is a sigmoid stretched to -1..1 and centred on 0, so a layer of tanh units passes on values that average near zero, and its slope reaches 1 at the centre. It still saturates: at 2.5 the output is 0.9866 and the slope only 0.0266. The red band marks every input where the slope is under 0.1, beyond about 1.8 on either side, and a unit whose inputs live there barely learns. The spec tanh_activation.t27 computes the curve, the slope and its own batch, tanh of 0, 1 and 2: 0, 0.762 and 0.964.
+
Try it
Drag x until the band starts and read the slope there; then compare the slope at 0 and at 2.5.
Drag along tanh: it is centred on zero, and where it flattens out its slope drops under 0.1. The red band shows every input where the gradient has all but stopped.
+
specs/ml/activation/tanh_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Derivative (can compute from output) | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: `math::exp` named a module this spec never imports, and
+// forward_batch allocated with `[]f32{input.len}`. tanh is now computed in the
+// form that cannot overflow: exp(2|x|) reaches inf in f32 at |x| = 45, and
+// inf / inf is NaN.
+
+module Tanh;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const TanhConfig = struct {
+ none : void,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // tanh(x) = (1 - e^(-2|x|)) / (1 + e^(-2|x|)), with the sign of x
+ fn forward(x: f32) -> f32 {
+ if (x < 0.0) {
+ return -forward(-x);
+ }
+ const t = exp(-2.0 * x);
+ return (1.0 - t) / (1.0 + t);
+ }
+
+ // forward_batch: output[i] = tanh(input[i])
+ fn forward_batch(input: []const f32, output: []f32) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = forward(input[i]);
+ i = i + 1;
+ }
+ }
+
+ // tanh'(x) = 1 - tanh(x)^2
+ fn derivative(x: f32) -> f32 {
+ const tanh_x = forward(x);
+ return 1.0 - (tanh_x * tanh_x);
+ }
+
+ // Given y = tanh(x), the derivative is 1 - y^2
+ fn derivative_from_output(tanh_x: f32) -> f32 {
+ return 1.0 - (tanh_x * tanh_x);
+ }
+
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [0.0, 1.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, &output);
+ return output[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // tanh(0) = 0, tanh(1) = 0.7615942, tanh is odd
+ test forward_basic_case
+ given input = 0.0
+ when result = forward(input)
+ then result == 0.0 and @abs(forward(1.0) - 0.7615942) < 0.000001 and forward(-1.0) == -forward(1.0)
+
+ // exp(2x) is inf in f32 here; the stable form still gives 1, not NaN
+ test forward_saturates_without_overflow
+ given input = 100.0
+ when result = forward(input)
+ then result == 1.0 and forward(-100.0) == -1.0
+
+ test forward_batch_basic_case
+ given first = batch_at(0)
+ when second = batch_at(1)
+ then first == 0.0 and second > 0.0 and second < 1.0 and batch_at(2) > second
+
+ test derivative_basic_case
+ given input = 0.0
+ when result = derivative(input)
+ then result == 1.0
+
+ test derivative_from_output_basic_case
+ given input = 0.5
+ when tanh_result = forward(input)
+ then derivative_from_output(tanh_result) > 0.0 and derivative_from_output(tanh_result) < 1.0 and derivative_from_output(tanh_result) == derivative(input)
+
+ test derivative_matches_finite_difference
+ given slope = (forward(0.5 + 0.001) - forward(0.5 - 0.001)) / 0.002
+ when result = derivative(0.5)
+ then @abs(result - slope) < 0.001
+
How the Huber loss treats a small error like squared error and a large one linearly, so outliers pull less.
+
Squared error lets one outlier dominate: an error of 3 costs 4.5 as half its square, as much as 36 errors of 0.5. Huber is half the square while |e| is within delta and then grows only linearly, delta |e| - delta^2 / 2, so with delta 1 that error costs 2.5. The two pieces meet with the same value and slope, and the gradient never grows past delta: a far-off point pulls on the model with a bounded force. For the spec's pair of errors 0.5 and 3 the mean is 1.3125, against 2.3125 for half the square. The spec huber_loss.t27 computes both.
+
Try it
Set delta to 0.5 and drag the error to 3; then find where the two curves part.
Drag the error along: Huber is half its square inside delta and only grows linearly outside, so one wild miss pulls on the model with a bounded force. The dashed curve is the pure square.
+
specs/ml/loss/huber_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Threshold where loss changes from quadratic to linear | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/huber_loss.tri: the port kept only
+// the first parameter and returned void, so forward computed a value and
+// threw it away, and every test asserted `result != undefined`.
+
+module HuberLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const HuberLossConfig = struct {
+ delta : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // per element e = prediction - target:
+ // |e| <= delta: 0.5 e^2 else: delta |e| - 0.5 delta^2
+ fn element_loss(e: f32, delta: f32) -> f32 {
+ const a = @abs(e);
+ if (a <= delta) {
+ return 0.5 * e * e;
+ }
+ return delta * a - 0.5 * delta * delta;
+ }
+
+ // forward(predictions, targets, delta) -> f32. The .tri gives the
+ // per-element formula only; this returns the MEAN over elements
+ // (PyTorch HuberLoss default reduction). 0 for n = 0.
+ fn forward(predictions: []const f32, targets: []const f32, delta: f32) -> f32 {
+ if (predictions.len == 0) {
+ return 0.0;
+ }
+ var sum_loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ sum_loss = sum_loss + element_loss(predictions[i] - targets[i], delta);
+ i = i + 1;
+ }
+ return sum_loss / (predictions.len as f32);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ fn huber_one(e: f32, delta: f32) -> f32 {
+ const predictions : [1]f32 = [e];
+ const targets : [1]f32 = [0.0];
+ return forward(&predictions, &targets, delta);
+ }
+
+ test quadratic_inside_delta
+ given result = huber_one(0.5, 1.0)
+ when negative = huber_one(-0.5, 1.0)
+ then @abs(result - 0.125) < 0.000001 and result == negative
+
+ // delta |e| - delta^2 / 2 = 3 - 0.5
+ test linear_outside_delta
+ given result = huber_one(3.0, 1.0)
+ when wide = huber_one(3.0, 2.0)
+ then @abs(result - 2.5) < 0.000001 and @abs(wide - 4.0) < 0.000001
+
+ // the two pieces meet with equal value and slope at |e| = delta
+ test continuous_at_delta
+ given below = huber_one(0.999, 1.0)
+ when above = huber_one(1.001, 1.0)
+ then @abs(above - below - 0.002) < 0.0001
+
+ // mean of (0.125, 2.5) over two elements
+ fn huber_two(e0: f32, e1: f32, delta: f32) -> f32 {
+ const predictions : [2]f32 = [e0 + 1.0, e1 + 1.0];
+ const targets : [2]f32 = [1.0, 1.0];
+ return forward(&predictions, &targets, delta);
+ }
+
+ test forward_mean
+ given result = huber_two(0.5, 3.0, 1.0)
+ when expected = 1.3125
+ then @abs(result - expected) < 0.000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant huber_zero_at_target
+ assert huber_one(0.0, 1.0) == 0.0
+
+
+
+
+
+
diff --git a/apps/website/public/learn/the-sigmoid/card.png b/apps/website/public/learn/the-sigmoid/card.png
new file mode 100644
index 0000000000..ee7366043c
Binary files /dev/null and b/apps/website/public/learn/the-sigmoid/card.png differ
diff --git a/apps/website/public/learn/the-sigmoid/index.html b/apps/website/public/learn/the-sigmoid/index.html
new file mode 100644
index 0000000000..79977d8eed
--- /dev/null
+++ b/apps/website/public/learn/the-sigmoid/index.html
@@ -0,0 +1,171 @@
+
+
+
+
+
+
+The sigmoid: a number into a probability: neural networks course, lesson 2 of 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiРусский
+
+
+
The sigmoid: a number into a probability
+
You will learn
How the sigmoid maps any input to a probability, and why its slope limits deep networks.
+
The sigmoid 1 / (1 + e^-x) squeezes the whole number line into 0 to 1, which is why it reads as a probability: 2 becomes 0.8808 and -4 becomes 0.018. Its slope is s (1 - s), computed from the output s itself, and it is largest at 0, where it is 0.25; at x = 8 it is 0.000335. A gradient that passes back through a stack of sigmoids is multiplied by at most 0.25 at each one, which is why deep sigmoid networks trained slowly. The spec sigmoid_activation.t27 computes the curve and the slope.
+
Try it
Find the input where the probability is 0.5; then drag to 6 and read how little slope is left.
Как рекуррентная ячейка переносит состояние от шага к шагу и почему эта память затухает.
+
Предложение приходит по одному слову, и рекуррентная ячейка переносит состояние h от шага к шагу: h' = tanh(W_x x + W_h h + b). В крошечной ячейке спеки W_x равен 1, а W_h 0.5, поэтому один импульс даёт 0.7616, 0.3634 и 0.1797: память о нём примерно вдвое слабеет каждый шаг. Второй импульс на шаге 3 поднимает h до 0.7968, чуть выше, чем начинался одиночный импульс, потому что старая память ещё там. После 8 шагов остаётся лишь 0.0466. Это затухание и есть причина, по которой простые ячейки забывают длинный контекст. Спека rnn_cell.t27 вычисляет каждый шаг.
+
Попробовать
Загрузите один импульс и прочитайте, за сколько шагов он падает ниже 0.1; затем сделайте вход чередующимся.
Tap the inputs of eight steps: the cell feeds its own last output back, h' = tanh(x + 0.5 h), so every pulse echoes on and fades by about half a step. Two pulses add up.
+
specs/ml/recurrent/rnn_cell.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Basic RNN recurrence equation | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written against undeclared `matmul`, `add`, `zeros`, `len` and `append`,
+// it applied tanh twice (h = tanh(tanh(z))), and the tests asserted
+// `result != undefined`. It is now the Elman cell,
+// h' = tanh(W_x x + W_h h + b), with W_x hidden x input and W_h hidden x hidden
+// (row-major), stepping over buffers the caller owns.
+
+module RnnCell;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_INPUT_SIZE : u32 = 128;
+ const DEFAULT_HIDDEN_SIZE : u32 = 128;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const RNNCellConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ };
+
+ pub const RNNState = struct {
+ hidden : []f32,
+ cell : []f32,
+ };
+
+ pub const RNNParams = struct {
+ w_x : []f32,
+ w_h : []f32,
+ b : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // new_hidden[i] = tanh(sum_j W_x[i][j] x[j] + sum_k W_h[i][k] h[k] + b[i])
+ fn forward_step(input: []const f32, hidden: []const f32, params: RNNParams, config: RNNCellConfig, new_hidden: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.hidden_size) {
+ var z : f32 = params.b[i];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + params.w_x[i * config.input_size + j] * input[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + params.w_h[i * config.hidden_size + k] * hidden[k];
+ k = k + 1;
+ }
+ new_hidden[i] = tanh(z);
+ i = i + 1;
+ }
+ }
+
+ // inputs: steps x input_size; outputs: steps x hidden_size, row t = h after step t.
+ // The state starts at initial_state and each step reads the previous row.
+ fn forward_sequence(inputs: []const f32, steps: usize, initial_state: []const f32, params: RNNParams, config: RNNCellConfig, outputs: []f32) -> void {
+ const n_in : usize = config.input_size;
+ const n_h : usize = config.hidden_size;
+ var t : usize = 0;
+ while (t < steps) {
+ if (t == 0) {
+ forward_step(inputs[0..n_in], initial_state, params, config, outputs[0..n_h]);
+ } else {
+ forward_step(inputs[t * n_in .. (t + 1) * n_in], outputs[(t - 1) * n_h .. t * n_h], params, config, outputs[t * n_h .. (t + 1) * n_h]);
+ }
+ t = t + 1;
+ }
+ }
+
+ // The first hidden_size entries of `buffer`, zeroed
+ fn init_state(buffer: []f32, hidden_size: u32) -> []f32 {
+ var i : usize = 0;
+ while (i < hidden_size) {
+ buffer[i] = 0.0;
+ i = i + 1;
+ }
+ return buffer[0..hidden_size];
+ }
+
+ // A 1-in, 1-hidden cell with W_x = 1, W_h = 0.5, b = 0: h' = tanh(x + 0.5 h)
+ fn tiny() -> RNNCellConfig {
+ return RNNCellConfig{ .input_size = 1, .hidden_size = 1 };
+ }
+
+ fn tiny_step(x: f32, h: f32) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const input : [1]f32 = [x];
+ const hidden : [1]f32 = [h];
+ var out : [1]f32 = [9.0];
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &out);
+ return out[0];
+ }
+
+ // The sequence (1, 0, 0) from h = 0: h after step t
+ fn tiny_sequence(t: usize) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const inputs : [3]f32 = [1.0, 0.0, 0.0];
+ const h0 : [1]f32 = [0.0];
+ var outputs : [3]f32 = [9.0] ** 3;
+ forward_sequence(&inputs, 3, &h0, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &outputs);
+ return outputs[t];
+ }
+
+ // A 2-in, 2-hidden cell: h' = tanh(W_x x + W_h h + b) with W_x = [[1, -1], [0.5, 0.5]],
+ // W_h = [[0.1, 0], [0, 0.1]], b = (0, -0.5), at x = (1, 2), h = (1, -1)
+ fn wide_step(index: usize) -> f32 {
+ var w_x : [4]f32 = [1.0, -1.0, 0.5, 0.5];
+ var w_h : [4]f32 = [0.1, 0.0, 0.0, 0.1];
+ var b : [2]f32 = [0.0, -0.5];
+ const input : [2]f32 = [1.0, 2.0];
+ const hidden : [2]f32 = [1.0, -1.0];
+ var out : [2]f32 = [9.0] ** 2;
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, RNNCellConfig{ .input_size = 2, .hidden_size = 2 }, &out);
+ return out[index];
+ }
+
+ fn init_len(hidden_size: u32) -> usize {
+ var buffer : [64]f32 = [9.0] ** 64;
+ const h = init_state(&buffer, hidden_size);
+ if (h[hidden_size - 1] != 0.0) {
+ return 0;
+ }
+ return h.len;
+ }
+
+ fn valid_input() -> f32 {
+ return tiny_step(1.0, 0.0);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // z = (1 - 2 + 0.1, 0.5 + 1 - 0.1 - 0.5) = (-0.9, 0.9): h' = (tanh(-0.9), tanh(0.9))
+ test forward_step_basic_case
+ given h0 = wide_step(0)
+ when h1 = wide_step(1)
+ then @abs(h0 + 0.7162979) < 0.000001 and @abs(h1 - 0.7162979) < 0.000001 and @abs(tiny_step(1.0, 0.0) - 0.7615942) < 0.000001
+
+ // The cell remembers: after the input pulse, h decays through the recurrent weight,
+ // h1 = tanh(1), h2 = tanh(0.5 h1), h3 = tanh(0.5 h2)
+ test forward_sequence_basic_case
+ given h1 = tiny_sequence(0)
+ when h3 = tiny_sequence(2)
+ then @abs(h1 - 0.7615942) < 0.000001 and @abs(tiny_sequence(1) - tanh(0.5 * h1)) < 0.000001 and @abs(h3 - tanh(0.5 * tiny_sequence(1))) < 0.000001 and h3 > 0.0 and h3 < tiny_sequence(1)
+
+ test init_state_basic_case
+ given hidden_size = 64
+ when result = init_len(hidden_size)
+ then result == hidden_size
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input_size >= 1
+ invariant rnn_cell_constraint_0
+ assert DEFAULT_INPUT_SIZE >= 1
+
+ // hidden_size >= 1
+ invariant rnn_cell_constraint_1
+ assert DEFAULT_HIDDEN_SIZE >= 1
+
+ // name: rnn_formula -- one tanh, not two: tanh(1) = 0.7615942, tanh(tanh(1)) = 0.6420
+ invariant rnn_cell_constraint_2
+ assert valid_input() > 0.76 and valid_input() < 0.77
+
+ // "Elman (1990) - Finding Structure in Time": the state carries the past forward
+ invariant rnn_cell_constraint_3
+ assert tiny_step(0.0, 1.0) > 0.0
+
+ // "Goodfellow et al. (2016) - Sequence Modeling with RNNs and LSTMs": tanh bounds the state
+ invariant rnn_cell_constraint_4
+ assert tiny_step(100.0, 1.0) <= 1.0 and tiny_step(-100.0, -1.0) >= -1.0
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/adam/card.png b/apps/website/public/ru/learn/adam/card.png
new file mode 100644
index 0000000000..ee1826e163
Binary files /dev/null and b/apps/website/public/ru/learn/adam/card.png differ
diff --git a/apps/website/public/ru/learn/adam/index.html b/apps/website/public/ru/learn/adam/index.html
new file mode 100644
index 0000000000..14f378e07b
--- /dev/null
+++ b/apps/website/public/ru/learn/adam/index.html
@@ -0,0 +1,232 @@
+
+
+
+
+
+
+Adam: момент и масштаб: курс о нейросетях, урок 18 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
Adam: момент и масштаб
+
Вы узнаете
Как Adam соединяет момент со скользящим масштабом и почему поправка смещения делает шаг 1 фиксированного размера.
+
Adam соединяет обе идеи: m, скользящее среднее градиента, как в моменте, и v, скользящее среднее его квадрата, как в RMSProp; шаг равен lr m / sqrt(v). Оба начинают с 0 и поначалу слишком малы, поэтому Adam делит их на 1 - beta^t, и на шаге 1 это делает сдвиг ровно lr, умноженным на знак градиента: градиенты 2, -40 и 0.5 сдвигают свои веса на одинаковые 0.001. На чаше спеки (x - 3)^2 при lr 0.1 Adam поднимается от 0, перелетает примерно до 3.18 и успокаивается на 3.00005 к шагу 200. Спека adam.t27 вычисляет каждый шаг.
+
Попробовать
Дайте трём градиентам очень разные размеры и прочитайте веса после шага 1; затем найдите, где путь достигает вершины.
Watch Adam carry a weight from 0 to the minimum at 3, then set three gradients of any size: the bias correction makes the first step lr times their sign, whatever their size.
+
specs/ml/optimizer/adam.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Adam adapts learning rate per parameter | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: step only checked its gradients with an undeclared `math::is_finite`
+// and updated nothing; the tests asserted `result != undefined`. step is now
+// Adam (Kingma and Ba, 2015) with bias correction. It refuses (returns false,
+// changes nothing) when a gradient is NaN or infinite, which is what the old body
+// validated.
+
+module Adam;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_BETA1 : f32 = 0.9;
+ const DEFAULT_BETA2 : f32 = 0.999;
+ const DEFAULT_EPSILON : f32 = 1e-8;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const AdamConfig = struct {
+ learning_rate : f32,
+ beta1 : f32,
+ beta2 : f32,
+ epsilon : f32,
+ weight_decay : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // b^t for a whole t >= 0
+ fn powi(b: f32, t: u32) -> f32 {
+ var r : f32 = 1.0;
+ var n : u32 = 0;
+ while (n < t) {
+ r = r * b;
+ n = n + 1;
+ }
+ return r;
+ }
+
+ // NaN is the one value not equal to itself; inf - inf is NaN
+ fn is_finite(x: f32) -> bool {
+ return x == x and (x - x) == (x - x);
+ }
+
+ // One Adam step number t (1-based) over params, with moment buffers m and v:
+ // g = grad + wd p
+ // m = b1 m + (1 - b1) g v = b2 v + (1 - b2) g^2
+ // m_hat = m / (1 - b1^t) v_hat = v / (1 - b2^t)
+ // p = p - lr m_hat / (sqrt(v_hat) + eps)
+ fn step(params: []f32, grads: []const f32, m: []f32, v: []f32, t: u32, config: AdamConfig) -> bool {
+ var i : usize = 0;
+ while (i < grads.len) {
+ if (!is_finite(grads[i])) {
+ return false;
+ }
+ i = i + 1;
+ }
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ i = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ m[i] = config.beta1 * m[i] + (1.0 - config.beta1) * g;
+ v[i] = config.beta2 * v[i] + (1.0 - config.beta2) * g * g;
+ const m_hat = m[i] / c1;
+ const v_hat = v[i] / c2;
+ params[i] = params[i] - config.learning_rate * m_hat / (sqrt(v_hat) + config.epsilon);
+ i = i + 1;
+ }
+ return true;
+ }
+
+ fn cfg(lr: f32) -> AdamConfig {
+ return AdamConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.0 };
+ }
+
+ // which: 0 = param, 1 = m, 2 = v after one step on params (1, 2, 3) with grads (g0, g1, g2)
+ fn first_step(g0: f32, g1: f32, g2: f32, which: u8, index: usize) -> f32 {
+ var params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [g0, g1, g2];
+ var m : [3]f32 = [0.0] ** 3;
+ var v : [3]f32 = [0.0] ** 3;
+ _ = step(¶ms, &grads, &m, &v, 1, cfg(0.001));
+ if (which == 1) {
+ return m[index];
+ }
+ if (which == 2) {
+ return v[index];
+ }
+ return params[index];
+ }
+
+ // a / b at run time, where IEEE gives NaN for 0 / 0 and inf for 1 / 0
+ fn ratio(a: f32, b: f32) -> f32 {
+ return a / b;
+ }
+
+ fn accepts(g: f32) -> bool {
+ var params : [1]f32 = [1.0];
+ const grads : [1]f32 = [g];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ return step(¶ms, &grads, &m, &v, 1, cfg(0.001)) and params[0] != 1.0;
+ }
+
+ // Minimise (x - 3)^2 from 0
+ fn descend(steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ var t : u32 = 1;
+ while (t <= steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ _ = step(¶ms, &grads, &m, &v, t, cfg(0.1));
+ t = t + 1;
+ }
+ return params[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Bias correction makes the first step lr * sign(g), whatever |g| is
+ test step_basic_case
+ given p0 = first_step(0.1, -5.0, 0.0, 0, 0)
+ when p1 = first_step(0.1, -5.0, 0.0, 0, 1)
+ then @abs(p0 - 0.999) < 0.000001 and @abs(p1 - 2.001) < 0.000001 and first_step(0.1, -5.0, 0.0, 0, 2) == 3.0
+
+ // m = 0.1 g and v = 0.001 g^2 after the first step
+ test step_with_gradients
+ given m2 = first_step(0.1, 0.2, 0.3, 1, 2)
+ when v2 = first_step(0.1, 0.2, 0.3, 2, 2)
+ then @abs(m2 - 0.03) < 0.0000001 and @abs(v2 - 0.00009) < 0.0000001
+
+ // Finite gradients are applied; NaN and inf are refused
+ test step_validates_gradients
+ given ok = accepts(1.5) and accepts(-2.3) and accepts(4.7)
+ when refused = !accepts(ratio(0.0, 0.0)) and !accepts(ratio(1.0, 0.0))
+ then ok and refused
+
+ test step_descends_a_quadratic
+ given result = descend(500)
+ then @abs(result - 3.0) < 0.01
+
+ invariant adam_betas_are_below_one
+ assert DEFAULT_BETA1 < 1.0 and DEFAULT_BETA2 < 1.0 and DEFAULT_BETA1 < DEFAULT_BETA2 and DEFAULT_EPSILON > 0.0
+
Что один блок энкодера трансформера делает со своим входом и что меняет перенос нормализаций вперёд.
+
Слой трансформера это два подслоя, каждый обёрнут остаточным сложением и нормализацией слоя: самовнимание смешивает позиции, затем сеть прямого распространения ширины 8 работает с каждой позицией отдельно. В порядке post-norm из исходной статьи блок кончается нормализацией, поэтому у каждой строки выхода среднее 0 и дисперсия 1.000. Pre-norm ставит нормализации перед каждым подслоем и кончается остаточным сложением, поэтому его строки выходят с дисперсиями 2.73, 3.33 и 0.96; глубокие стопки так учатся легче, с одной нормализацией в самом верху. Спека encoder_block.t27 прогоняет один блок на 3 позициях, d_model 4 и 2 головах.
+
Попробовать
Переключайтесь между post-norm и pre-norm и читайте дисперсии строк; затем найдите позицию, которую голова 1 читает больше всего.
Three positions through one transformer block: attention, a residual, a feed-forward net and layer norm. Post-norm ends every row at mean 0 and variance 1; pre-norm does not.
+
specs/ml/transformer/encoder_block.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Standard Transformer encoder block with residual connections | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: attention and the FFN returned their input ("in real implementation
+// ..."), forward skipped both residuals, and the tests asked only for a non-empty
+// output from undeclared `default_input()`. The block now computes the Transformer
+// encoder layer over a seq x d_model input. Attention is multi-head scaled
+// dot-product, softmax(q k^T / sqrt(d_head)) v per head, with identity projections.
+// Then come the position-wise FFN and two residual + layer-norm steps: post-norm
+// LN(x + f(x)) by default, pre-norm x + f(LN(x)) when use_pre_norm is set.
+
+module EncoderBlock;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_D_MODEL : u32 = 512;
+ const DEFAULT_N_HEADS : u32 = 8;
+ const DEFAULT_D_FF : u32 = 2048;
+ const DEFAULT_DROPOUT : f32 = 0.1;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const EncoderBlockConfig = struct {
+ d_model : u32,
+ n_heads : u32,
+ d_ff : u32,
+ dropout : f32,
+ use_pre_norm : bool,
+ };
+
+ pub const FFNWeights = struct {
+ w1 : []const f32,
+ b1 : []const f32,
+ w2 : []const f32,
+ b2 : []const f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // q, k, v and out are seq x d_model; attn_weights is n_heads x seq x seq. Head h uses
+ // columns h * d_head .. (h + 1) * d_head.
+ fn multi_head_attention(query: []const f32, key: []const f32, value: []const f32, seq: usize, n_heads: u32, d_model: u32, out: []f32, attn_weights: []f32) -> void {
+ const d_head : usize = d_model / n_heads;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d_head)));
+ var h : usize = 0;
+ while (h < n_heads) {
+ var i : usize = 0;
+ while (i < seq) {
+ const row = attn_weights[(h * seq + i) * seq .. (h * seq + i + 1) * seq];
+ var max_s : f32 = -1.0e30;
+ var j : usize = 0;
+ while (j < seq) {
+ var dot : f32 = 0.0;
+ var c : usize = 0;
+ while (c < d_head) {
+ dot = dot + query[i * d_model + h * d_head + c] * key[j * d_model + h * d_head + c];
+ c = c + 1;
+ }
+ row[j] = dot * scale;
+ max_s = @max(max_s, row[j]);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ var c : usize = 0;
+ while (c < d_head) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ acc = acc + row[j] / sum * value[j * d_model + h * d_head + c];
+ j = j + 1;
+ }
+ out[i * d_model + h * d_head + c] = acc;
+ c = c + 1;
+ }
+ j = 0;
+ while (j < seq) {
+ row[j] = row[j] / sum;
+ j = j + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ // out = W2 relu(W1 x + b1) + b2 for one row; hidden is d_ff floats of scratch.
+ // Dropout is the identity at inference, which is what this block computes.
+ fn feed_forward(input: []const f32, weights: FFNWeights, d_model: u32, d_ff: u32, hidden: []f32, out: []f32) -> void {
+ var f : usize = 0;
+ while (f < d_ff) {
+ var z : f32 = weights.b1[f];
+ var j : usize = 0;
+ while (j < d_model) {
+ z = z + weights.w1[f * d_model + j] * input[j];
+ j = j + 1;
+ }
+ hidden[f] = @max(z, 0.0);
+ f = f + 1;
+ }
+ var o : usize = 0;
+ while (o < d_model) {
+ var y : f32 = weights.b2[o];
+ f = 0;
+ while (f < d_ff) {
+ y = y + weights.w2[o * d_ff + f] * hidden[f];
+ f = f + 1;
+ }
+ out[o] = y;
+ o = o + 1;
+ }
+ }
+
+ // (x - mean) / sqrt(var + 1e-5), no learned scale or shift
+ fn layer_norm(x: []const f32, out: []f32) -> void {
+ var mean : f32 = 0.0;
+ var k : usize = 0;
+ while (k < x.len) {
+ mean = mean + x[k];
+ k = k + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(x.len));
+ var variance : f32 = 0.0;
+ k = 0;
+ while (k < x.len) {
+ variance = variance + (x[k] - mean) * (x[k] - mean);
+ k = k + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(x.len));
+ k = 0;
+ while (k < x.len) {
+ out[k] = (x[k] - mean) / sqrt(variance + 0.00001);
+ k = k + 1;
+ }
+ }
+
+ // input and out are seq x d_model. scratch needs 3 seq d_model + n_heads seq^2 + d_ff + d_model floats.
+ fn forward(input: []const f32, seq: usize, ffn: FFNWeights, config: EncoderBlockConfig, out: []f32, scratch: []f32) -> void {
+ const n : usize = seq * config.d_model;
+ const d : usize = config.d_model;
+ const normed = scratch[0..n];
+ const attn = scratch[n .. 2 * n];
+ const mid = scratch[2 * n .. 3 * n];
+ const weights = scratch[3 * n .. 3 * n + config.n_heads * seq * seq];
+ const rest = 3 * n + config.n_heads * seq * seq;
+ const hidden = scratch[rest .. rest + config.d_ff];
+ const row = scratch[rest + config.d_ff .. rest + config.d_ff + d];
+ var t : usize = 0;
+ if (config.use_pre_norm) {
+ while (t < seq) {
+ layer_norm(input[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ multi_head_attention(normed, normed, normed, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ mid[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(mid[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ feed_forward(normed[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ out[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ t = t + 1;
+ }
+ return;
+ }
+ multi_head_attention(input, input, input, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ normed[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(normed[t * d .. (t + 1) * d], mid[t * d .. (t + 1) * d]);
+ feed_forward(mid[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ normed[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ layer_norm(normed[t * d .. (t + 1) * d], out[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads, d_ff 8
+ fn cfg(pre_norm: bool) -> EncoderBlockConfig {
+ return EncoderBlockConfig{ .d_model = 4, .n_heads = 2, .d_ff = 8, .dropout = DEFAULT_DROPOUT, .use_pre_norm = pre_norm };
+ }
+
+ fn fixture_input(x: []f32) -> void {
+ const v : [12]f32 = [1.0, 0.0, 0.5, -1.0, 0.0, 1.0, -0.5, 2.0, 1.0, 1.0, 0.0, 0.0];
+ var k : usize = 0;
+ while (k < 12) {
+ x[k] = v[k];
+ k = k + 1;
+ }
+ }
+
+ // which: 0 = out[index], 1 = attn_weights[index] of self-attention on the fixture
+ fn attention_at(which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var out : [12]f32 = [9.0] ** 12;
+ var weights : [18]f32 = [9.0] ** 18;
+ multi_head_attention(&x, &x, &x, 3, 2, 4, &out, &weights);
+ if (which == 0) {
+ return out[index];
+ }
+ return weights[index];
+ }
+
+ // Two keys equal: attention splits evenly and returns the mean of the values
+ fn tie_attention(index: usize) -> f32 {
+ const q : [2]f32 = [1.0, 1.0];
+ const k : [2]f32 = [2.0, 2.0];
+ const v : [2]f32 = [4.0, 10.0];
+ var out : [2]f32 = [0.0] ** 2;
+ var weights : [4]f32 = [0.0] ** 4;
+ multi_head_attention(&q, &k, &v, 2, 1, 1, &out, &weights);
+ return out[index];
+ }
+
+ fn ffn_fill(w1: []f32, b1: []f32, w2: []f32, b2: []f32) -> void {
+ var k : usize = 0;
+ while (k < 32) {
+ w1[k] = @as(f32, @floatFromInt((k * 5 + 1) % 9)) / 8.0 - 0.5;
+ w2[k] = @as(f32, @floatFromInt((k * 7 + 2) % 9)) / 8.0 - 0.5;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 8) {
+ b1[k] = 0.1;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 4) {
+ b2[k] = -0.1;
+ k = k + 1;
+ }
+ }
+
+ // which: 0 = out[index]; 1 = mean of row `index`; 2 = variance of row `index`
+ fn block_at(pre_norm: bool, which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ var out : [12]f32 = [9.0] ** 12;
+ var scratch : [3 * 12 + 18 + 8 + 4]f32 = undefined;
+ forward(&x, 3, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, cfg(pre_norm), &out, &scratch);
+ if (which == 0) {
+ return out[index];
+ }
+ var mean : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ mean = mean + out[index * 4 + c] / 4.0;
+ c = c + 1;
+ }
+ if (which == 1) {
+ return mean;
+ }
+ var variance : f32 = 0.0;
+ c = 0;
+ while (c < 4) {
+ variance = variance + (out[index * 4 + c] - mean) * (out[index * 4 + c] - mean) / 4.0;
+ c = c + 1;
+ }
+ return variance;
+ }
+
+ // FFN on row (1, 0, 0.5, -1) of the fixture weights; out[index]
+ fn ffn_at(index: usize) -> f32 {
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ const x : [4]f32 = [1.0, 0.0, 0.5, -1.0];
+ var hidden : [8]f32 = undefined;
+ var out : [4]f32 = [9.0] ** 4;
+ feed_forward(&x, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, 4, 8, &hidden, &out);
+ return out[index];
+ }
+
+ fn valid_input() -> u32 {
+ return DEFAULT_D_MODEL / DEFAULT_N_HEADS;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Every attention row is a distribution; equal keys give the mean of the values
+ test multi_head_attention_basic_case
+ given row_sum = attention_at(1, 0) + attention_at(1, 1) + attention_at(1, 2)
+ when tied = tie_attention(0)
+ then @abs(row_sum - 1.0) < 0.000001 and tied == 7.0 and tie_attention(1) == 7.0 and attention_at(0, 0) != 9.0
+
+ // Heads are independent: head 1's weights (rows 9..17) differ from head 0's
+ test multi_head_attention_heads_differ
+ given w0 = attention_at(1, 1)
+ when w1 = attention_at(1, 10)
+ then w0 != w1 and w0 > 0.0 and w1 > 0.0
+
+ // W2 relu(W1 x + b1) + b2 on the fixture row, worked by hand: (0.4125, ..., -0.178125)
+ test feed_forward_basic_case
+ given y0 = ffn_at(0)
+ when y3 = ffn_at(3)
+ then @abs(y0 - 0.4125) < 0.000001 and @abs(y3 + 0.178125) < 0.000001 and @abs(ffn_at(2) + 0.928125) < 0.000001
+
+ // Post-norm: every output row has mean 0 and variance 1
+ test forward_basic_case
+ given mean1 = block_at(false, 1, 1)
+ when var1 = block_at(false, 2, 1)
+ then @abs(mean1) < 0.00001 and @abs(var1 - 1.0) < 0.001 and @abs(block_at(false, 1, 2)) < 0.00001
+
+ // Pre-norm keeps the residual stream un-normalised, so it differs from post-norm
+ test forward_pre_norm_differs
+ given pre = block_at(true, 0, 0)
+ when post = block_at(false, 0, 0)
+ then pre != post and pre == pre
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // d_model % n_heads == 0
+ invariant encoder_block_constraint_0
+ assert DEFAULT_D_MODEL % DEFAULT_N_HEADS == 0 and cfg(false).d_model % cfg(false).n_heads == 0
+
+ // d_ff >= d_model
+ invariant encoder_block_constraint_1
+ assert DEFAULT_D_FF >= DEFAULT_D_MODEL and cfg(false).d_ff >= cfg(false).d_model
+
+ // name: encoder_block -- each head gets d_model / n_heads columns
+ invariant encoder_block_constraint_2
+ assert valid_input() == 64
+
+ // "Vaswani et al. (2017) - Attention Is All You Need": dropout is a probability
+ invariant encoder_block_constraint_3
+ assert DEFAULT_DROPOUT >= 0.0 and DEFAULT_DROPOUT < 1.0
+
+ // "Baevski et al. (2020) - Revisiting the Self-Attention": both norm placements are offered
+ invariant encoder_block_constraint_4
+ assert cfg(true).use_pre_norm and !cfg(false).use_pre_norm
+
Он продолжает курс о шинах. Каждый виджет в нём исполняет тела fn спеки t27, скомпилированные в WebAssembly: двигаете ползунок — отвечает спека.
Курс 9, 27 уроков, в каждом живой виджет и спека t27: сумматоры и цепи переноса, умножители и слайс DSP48E1, фиксированная точка, CORDIC, дискретизация, фильтры FIR и CIC, БПФ и фильтр на стенде. Каждое число считает скомпилированная спека.
diff --git a/apps/website/public/ru/learn/attention/card.png b/apps/website/public/ru/learn/attention/card.png
new file mode 100644
index 0000000000..6606b8a835
Binary files /dev/null and b/apps/website/public/ru/learn/attention/card.png differ
diff --git a/apps/website/public/ru/learn/attention/index.html b/apps/website/public/ru/learn/attention/index.html
new file mode 100644
index 0000000000..9de3eed72f
--- /dev/null
+++ b/apps/website/public/ru/learn/attention/index.html
@@ -0,0 +1,382 @@
+
+
+
+
+
+
+Внимание: курс о нейросетях, урок 24 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
Внимание
+
Вы узнаете
Как внимание сравнивает каждый запрос с каждым ключом, что даёт softmax по каждой строке и что убирает причинная маска.
+
Внимание позволяет каждой позиции решить, какие другие позиции читать. Каждый запрос сравнивается с каждым ключом, q . k / sqrt(d), и softmax превращает каждую строку оценок в веса с суммой 1; выход это значения, усреднённые этими весами. Две головы делают это на двух половинах вектора. С причинной маской позиция не может читать будущее: в голове 0 позиция 0 читает только себя, позиция 1 делит 0.452 и 0.548, а позиция 2 распределяет 0.230, 0.321 и 0.449. Без маски каждая строка распределяется по всем трём ключам, от 0.21 до 0.47. Спека attention_mechanism.t27 вычисляет каждый вес на своей фикстуре.
+
Попробовать
Выключите и включите маску и прочитайте строку 0; затем сравните две головы в строке 2.
Two heads of attention over three positions: each row is how much one query reads each key. Switch on the causal mask and a position can no longer read the future.
+
specs/ml/recurrent/attention_mechanism.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Memory-efficient attention for long sequences | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: redeclared `var i` in one scope four times, called `@infinity()` and
+// `@random()` (not Zig), and indexed every row as head_dim wide inside d_model-wide
+// rows. backward also skipped the softmax Jacobian: it pushed dL/dweights straight
+// into the scores. It is now multi-head scaled dot-product attention over seq x
+// d_model rows. Head h reads columns h*hd .. (h+1)*hd, the weights are
+// heads x seq x seq, and causal masking is optional. backward is the exact
+// gradient, checked against finite differences. Dropout is carried in the config
+// and not applied: this is the inference computation.
+
+module Attention;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const NEG_INF : f32 = -1.0e30;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const AttentionConfig = struct {
+ d_model : u32,
+ num_heads : u32,
+ dropout : f32,
+ causal : bool,
+ };
+
+ pub const AttentionOutput = struct {
+ output : []f32,
+ weights : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn seq_of(query: []const f32, config: AttentionConfig) -> usize {
+ return query.len / config.d_model;
+ }
+
+ fn scale_of(config: AttentionConfig) -> f32 {
+ return 1.0 / sqrt(@as(f32, @floatFromInt(config.d_model / config.num_heads)));
+ }
+
+ fn forward(query: []const f32, key: []const f32, value: []const f32, output: []f32, weights: []f32, config: AttentionConfig) -> void {
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const row = weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var max_s : f32 = NEG_INF;
+ var j : usize = 0;
+ while (j < n) {
+ var score : f32 = NEG_INF;
+ if (!(config.causal and j > i)) {
+ score = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ score = score + query[i * d + h * hd + k] * key[j * d + h * hd + k];
+ k = k + 1;
+ }
+ score = score * scale;
+ }
+ row[j] = score;
+ max_s = @max(max_s, score);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ j = 0;
+ while (j < n) {
+ row[j] = row[j] / sum;
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < hd) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ acc = acc + row[j] * value[j * d + h * hd + k];
+ j = j + 1;
+ }
+ output[i * d + h * hd + k] = acc;
+ k = k + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ // dV_j += P_ij dO_i; dP_ij = dO_i . V_j; dS_ij = P_ij (dP_ij - sum_k P_ik dP_ik);
+ // dQ_i += scale dS_ij K_j; dK_j += scale dS_ij Q_i (per head; masked P_ij are 0)
+ fn backward(grad_output: []const f32, query: []const f32, key: []const f32, value: []const f32, attn_weights: []f32, grad_query: []f32, grad_key: []f32, grad_value: []f32, config: AttentionConfig) -> void {
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var x : usize = 0;
+ while (x < n * d) {
+ grad_query[x] = 0.0;
+ grad_key[x] = 0.0;
+ grad_value[x] = 0.0;
+ x = x + 1;
+ }
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const p = attn_weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var row_dot : f32 = 0.0;
+ var j : usize = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ grad_value[j * d + h * hd + k] = grad_value[j * d + h * hd + k] + p[j] * grad_output[i * d + h * hd + k];
+ k = k + 1;
+ }
+ row_dot = row_dot + p[j] * dp;
+ j = j + 1;
+ }
+ j = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ k = k + 1;
+ }
+ const ds = p[j] * (dp - row_dot) * scale;
+ k = 0;
+ while (k < hd) {
+ grad_query[i * d + h * hd + k] = grad_query[i * d + h * hd + k] + ds * key[j * d + h * hd + k];
+ grad_key[j * d + h * hd + k] = grad_key[j * d + h * hd + k] + ds * query[i * d + h * hd + k];
+ k = k + 1;
+ }
+ j = j + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ fn cfg(causal: bool, dropout: f32) -> AttentionConfig {
+ return AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = dropout, .causal = causal };
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads (columns 0-1 and 2-3); q / k / v scaled down so the
+ // softmax is not saturated. which: 0 = output, 1 = weights.
+ fn fixture(q: []f32, k: []f32, v: []f32) -> void {
+ var x : usize = 0;
+ while (x < 12) {
+ q[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ k[x] = @as(f32, @floatFromInt(x / 4 + 1)) / 4.0 - @as(f32, @floatFromInt(x % 2)) / 3.0;
+ v[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
+ }
+ }
+
+ fn forward_at(causal: bool, which: u8, index: usize) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = [9.0] ** 12;
+ var w : [18]f32 = [9.0] ** 18;
+ forward(&q, &k, &v, &out, &w, cfg(causal, 0.0));
+ if (which == 0) {
+ return out[index];
+ }
+ return w[index];
+ }
+
+ // L = sum_x g_x O_x with g_x = (x + 1) / 10, one entry of q, k or v moved by delta
+ // (which: 0 = q, 1 = k, 2 = v)
+ fn loss(which: u8, index: usize, delta: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ if (which == 0) {
+ q[index] = q[index] + delta;
+ }
+ if (which == 1) {
+ k[index] = k[index] + delta;
+ }
+ if (which == 2) {
+ v[index] = v[index] + delta;
+ }
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, 0.0));
+ var total : f32 = 0.0;
+ var x : usize = 0;
+ while (x < 12) {
+ total = total + @as(f32, @floatFromInt(x + 1)) / 10.0 * out[x];
+ x = x + 1;
+ }
+ return total;
+ }
+
+ fn grad(which: u8, index: usize, dropout: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, dropout));
+ var g : [12]f32 = undefined;
+ var x : usize = 0;
+ while (x < 12) {
+ g[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
+ }
+ var gq : [12]f32 = undefined;
+ var gk : [12]f32 = undefined;
+ var gv : [12]f32 = undefined;
+ backward(&g, &q, &k, &v, &w, &gq, &gk, &gv, cfg(false, dropout));
+ if (which == 0) {
+ return gq[index];
+ }
+ if (which == 1) {
+ return gk[index];
+ }
+ return gv[index];
+ }
+
+ fn slope(which: u8, index: usize) -> f32 {
+ return (loss(which, index, 0.001) - loss(which, index, -0.001)) / 0.002;
+ }
+
+ fn valid_input() -> u32 {
+ return cfg(false, 0.0).d_model % cfg(false, 0.0).num_heads;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Each weight row is a distribution, and each output is a weighted mean of the values
+ // (so it lies between the smallest and largest value in its column)
+ test forward_basic_case {
+ const row_sum = forward_at(false, 1, 0) + forward_at(false, 1, 1) + forward_at(false, 1, 2);
+ assert(@abs(row_sum - 1.0) < 0.000001);
+ assert(forward_at(false, 0, 0) + forward_at(false, 0, 1) + forward_at(false, 0, 2) + forward_at(false, 0, 3) > 0.0); // computed a meaningful output
+ assert(forward_at(false, 0, 0) > 0.1 and forward_at(false, 0, 0) < 0.9);
+ }
+
+ // dQ, dK and dV against central differences of L = g . O
+ test backward_basic_case {
+ assert(@abs(grad(0, 0, 0.0) - slope(0, 0)) < 0.001);
+ assert(@abs(grad(0, 7, 0.0) - slope(0, 7)) < 0.001);
+ assert(@abs(grad(1, 5, 0.0) - slope(1, 5)) < 0.001);
+ assert(@abs(grad(2, 10, 0.0) - slope(2, 10)) < 0.001);
+ assert(grad(0, 0, 0.0) + grad(0, 1, 0.0) + grad(0, 2, 0.0) + grad(0, 3, 0.0) != 0.0); // computed gradients
+ }
+
+ // Position 0 attends only to itself, position 1 to positions 0 and 1
+ test forward_causal_masking {
+ assert(forward_at(true, 1, 0) == 1.0);
+ assert(forward_at(true, 1, 1) == 0.0 and forward_at(true, 1, 2) == 0.0);
+ assert(forward_at(true, 1, 1 * 3 + 2) < 0.999);
+ assert(forward_at(true, 1, 1 * 3 + 2) == 0.0);
+ assert(forward_at(true, 0, 0) == 0.1);
+ }
+
+ // Dropout is not applied here (inference), so it leaves the gradients unchanged
+ test backward_dropout {
+ assert(grad(2, 3, 0.5) == grad(2, 3, 0.0));
+ assert(grad(0, 11, 0.5) == grad(0, 11, 0.0));
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // d_model % num_heads == 0
+ invariant attention_constraint_0
+ assert valid_input() == 0
+
+ // 0 <= dropout < 1
+ invariant attention_constraint_1
+ assert cfg(false, 0.5).dropout >= 0.0 and cfg(false, 0.5).dropout < 1.0
+
+ // query.shape == key.shape == value.shape: seq is read from the query
+ invariant attention_constraint_2
+ assert cfg(false, 0.0).d_model == 4
+
Как градиент каждого веса и каждого входа слоя следует из градиента его выходов.
+
Чтобы учиться, каждому весу нужно знать, как меняется потеря, когда он сдвигается. Обратное распространение отвечает по одному слою за раз: при данном g, насколько потеря хочет сдвинуть каждый выход, вес от входа i к выходу o получает g_o x_i, а вход i получает сумму по o от W[o][i] g_o и передаёт её слою ниже. При входах (1, 2, 3) и g = (1, 0.5) градиенты весов равны 1, 2, 3 и 0.5, 1, 1.5, а градиенты входов 0.5, 2 и 3.5. Слой линейный, поэтому сдвиг x0 на 1 сдвигает потерю g . y с 15.25 до 15.75, на его градиент 0.5. Спека dense_layer.t27 вычисляет все три.
+
Попробовать
Поставьте g = (1, 0) и посмотрите, какие рёбра гаснут; затем поставьте g = (-2, 1) и проверьте градиенты входов вручную.
Set how much the loss wants each of two outputs to move: every weight gets that times its input, every input gets W transposed times it. The edges and nodes show the numbers.
+
specs/ml/layers/dense_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Row-major weight layout for cache efficiency | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward and backward were empty placeholders, the tests asserted
+// `result != undefined`, and the invariants asserted `true` over an undeclared
+// `valid_input`. Dense is now y = act(W x + b) with W row-major
+// (output_size x input_size), and backward gives the three gradients.
+
+module Dense;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const MIN_INPUT_SIZE : u32 = 1;
+ const MAX_INPUT_SIZE : u32 = 4096;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const DenseConfig = struct {
+ input_size : u32,
+ output_size : u32,
+ has_bias : bool,
+ activation : ActivationType,
+ };
+
+ pub const ActivationType = enum(u8) {
+ None,
+ ReLU,
+ GELU,
+ Sigmoid,
+ Tanh,
+ Softmax,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // relu, the one activation the tests exercise besides None
+ fn activate(z: f32, activation: ActivationType) -> f32 {
+ if (activation == ActivationType.ReLU) {
+ if (z > 0.0) {
+ return z;
+ }
+ return 0.0;
+ }
+ return z;
+ }
+
+ // output[o] = act(sum_i weights[o * in + i] * input[i] + bias[o])
+ fn forward(input: []const f32, weights: []const f32, bias: []const f32, config: DenseConfig, output: []f32) -> void {
+ var o : usize = 0;
+ while (o < config.output_size) {
+ var z : f32 = 0.0;
+ if (config.has_bias) {
+ z = bias[o];
+ }
+ var i : usize = 0;
+ while (i < config.input_size) {
+ z = z + weights[o * config.input_size + i] * input[i];
+ i = i + 1;
+ }
+ output[o] = activate(z, config.activation);
+ o = o + 1;
+ }
+ }
+
+ // For a linear layer (activation None), given dL/dy = grad_output:
+ // grad_input[i] = sum_o W[o][i] grad_output[o] (W^T g)
+ // grad_weights[o][i] = grad_output[o] * input[i] (g x^T)
+ // grad_bias[o] = grad_output[o]
+ fn backward(grad_output: []const f32, input: []const f32, weights: []const f32, config: DenseConfig, grad_input: []f32, grad_weights: []f32, grad_bias: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.input_size) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var o : usize = 0;
+ while (o < config.output_size) {
+ grad_bias[o] = grad_output[o];
+ i = 0;
+ while (i < config.input_size) {
+ grad_input[i] = grad_input[i] + weights[o * config.input_size + i] * grad_output[o];
+ grad_weights[o * config.input_size + i] = grad_output[o] * input[i];
+ i = i + 1;
+ }
+ o = o + 1;
+ }
+ }
+
+ // The fixture: 3 inputs, 2 outputs, W = [[1, 2, 3], [-1, 0, 1]], b = [0.5, -0.5]
+ fn cfg(activation: ActivationType) -> DenseConfig {
+ return DenseConfig{ .input_size = 3, .output_size = 2, .has_bias = true, .activation = activation };
+ }
+
+ fn forward_at(x0: f32, x1: f32, x2: f32, activation: ActivationType, index: usize) -> f32 {
+ const input : [3]f32 = [x0, x1, x2];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ const bias : [2]f32 = [0.5, -0.5];
+ var output : [2]f32 = [0.0] ** 2;
+ forward(&input, &weights, &bias, cfg(activation), &output);
+ return output[index];
+ }
+
+ // L = g . y for g = (g0, g1): the loss whose gradients backward computes
+ fn loss_at(x0: f32, x1: f32, x2: f32, g0: f32, g1: f32) -> f32 {
+ return g0 * forward_at(x0, x1, x2, ActivationType.None, 0) + g1 * forward_at(x0, x1, x2, ActivationType.None, 1);
+ }
+
+ // which: 0 = grad_input, 1 = grad_weights, 2 = grad_bias
+ fn backward_at(g0: f32, g1: f32, which: u8, index: usize) -> f32 {
+ const grad_output : [2]f32 = [g0, g1];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ var grad_weights : [6]f32 = [9.0] ** 6;
+ var grad_bias : [2]f32 = [9.0] ** 2;
+ backward(&grad_output, &input, &weights, cfg(ActivationType.None), &grad_input, &grad_weights, &grad_bias);
+ if (which == 0) {
+ return grad_input[index];
+ }
+ if (which == 1) {
+ return grad_weights[index];
+ }
+ return grad_bias[index];
+ }
+
+ // An empty layer (no inputs, no outputs) writes nothing
+ fn empty_forward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ forward(&none, &none, &none, empty_cfg, &sentinel);
+ return sentinel[0] == 7.0;
+ }
+
+ fn empty_backward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ backward(&none, &none, &none, empty_cfg, &sentinel, &sentinel, &sentinel);
+ return sentinel[0] == 7.0;
+ }
+
+ fn valid_input() -> f32 {
+ return forward_at(1.0, 2.0, 3.0, ActivationType.None, 0);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // x = (1, 2, 3): y0 = 1 + 4 + 9 + 0.5 = 14.5, y1 = -1 + 0 + 3 - 0.5 = 1.5
+ test forward_basic_case
+ given y0 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 0)
+ when y1 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 1)
+ then y0 == 14.5 and y1 == 1.5
+
+ // ReLU clips the negative pre-activation: x = (0, 0, -1) gives y = (-2.5, -1.5) -> (0, 0)
+ test forward_relu_clips_negatives
+ given linear = forward_at(0.0, 0.0, -1.0, ActivationType.None, 0)
+ when clipped = forward_at(0.0, 0.0, -1.0, ActivationType.ReLU, 0)
+ then linear == -2.5 and clipped == 0.0 and forward_at(1.0, 2.0, 3.0, ActivationType.ReLU, 0) == 14.5
+
+ test forward_empty_input
+ given result = empty_forward_untouched()
+ then result
+
+ // g = (1, 2): grad_input = W^T g = (1 - 2, 2 + 0, 3 + 2) = (-1, 2, 5); grad_bias = g;
+ // grad_weights[1][2] = g1 * x2 = 6
+ test backward_basic_case
+ given gi0 = backward_at(1.0, 2.0, 0, 0)
+ when gi2 = backward_at(1.0, 2.0, 0, 2)
+ then gi0 == -1.0 and backward_at(1.0, 2.0, 0, 1) == 2.0 and gi2 == 5.0 and backward_at(1.0, 2.0, 2, 1) == 2.0 and backward_at(1.0, 2.0, 1, 5) == 6.0
+
+ // dL/dx1 by finite difference equals grad_input[1]
+ test backward_matches_finite_difference
+ given slope = (loss_at(1.0, 2.0 + 0.01, 3.0, 1.0, 2.0) - loss_at(1.0, 2.0 - 0.01, 3.0, 1.0, 2.0)) / 0.02
+ when result = backward_at(1.0, 2.0, 0, 1)
+ then @abs(result - slope) < 0.001
+
+ test backward_empty_gradient
+ given result = empty_backward_untouched()
+ then result
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.size() == config.input_size: the bounds the layer accepts
+ invariant dense_constraint_0
+ assert MIN_INPUT_SIZE >= 1 and MAX_INPUT_SIZE >= MIN_INPUT_SIZE
+
+ // output.size() == config.output_size: each output is one row of W x + b
+ invariant dense_constraint_1
+ assert valid_input() == 14.5
+
+ // weights.size() == input_size * output_size: the fixture holds 3 * 2 = 6 weights
+ invariant dense_constraint_2
+ assert cfg(ActivationType.None).input_size * cfg(ActivationType.None).output_size == 6
+
+ // bias.size() == output_size (if has_bias): the fixture carries a bias of 2
+ invariant dense_constraint_3
+ assert cfg(ActivationType.None).has_bias
+
+ // All inputs must be finite (no NaN/Inf): finite in, finite out
+ invariant dense_constraint_4
+ assert valid_input() == valid_input()
+
Как пакетная нормализация приводит признаки разного масштаба к одному и чем она пользуется при выводе.
+
Признаки разного масштаба дёргают сеть: в пакете спеки один признак в десять раз больше другого. Пакетная нормализация вычитает среднее каждого признака по пакету и делит на его разброс, поэтому признаки 1, 2, 3, 4 и 10, 20, 30, 40 оба выходят как -1.34, -0.45, 0.45 и 1.34, а -1, -1, 1, 1 выходит как -1 и 1. Затем выученные gamma и beta масштабируют и сдвигают его обратно, где это полезно. При выводе слой вместо пакета использует скользящие средние: при среднем 0, дисперсии 1, gamma 1 и beta 0.25 входы 1, 2 и 3 дают 1.25, 2.25 и 3.25. Спека batchnorm_layer.t27 вычисляет оба режима.
+
Попробовать
Сравните два нормализованных столбца; затем поставьте скользящее среднее 2 и дисперсию 4 и прочитайте выходы вывода.
A batch of 4 samples whose 3 features live on scales 1, 10 and 1: batch norm gives each feature mean 0 and variance 1. Then set gamma and the running statistics for inference.
+
specs/ml/layers/batchnorm_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Prevent division by zero in sqrt | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward copied one element and backward zeroed two gradients
+// (placeholders), and the tests pinned those placeholders: output[0] == input[0],
+// grad_gamma[0] == grad_beta[0] == 0. They now compute batch normalisation:
+// forward/backward in inference form, normalising by the running statistics, and
+// forward_train normalising by the batch's own mean and variance.
+
+module Batchnorm;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_EPSILON : f32 = 1.0e-5;
+ const DEFAULT_MOMENTUM : f32 = 0.1;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const BatchNormConfig = struct {
+ num_features : u32,
+ epsilon : f32,
+ momentum : f32,
+ affine : bool,
+ track_running_stats : bool,
+ };
+
+ pub const BatchNormState = struct {
+ running_mean : []f32,
+ running_var : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Inference: output[f] = gamma[f] (input[f] - running_mean[f]) / sqrt(running_var[f] + eps) + beta[f]
+ fn forward(input: []const f32, gamma: []const f32, beta: []const f32, state: BatchNormState, output: []f32, config: BatchNormConfig) -> void {
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const x_hat = (input[f] - state.running_mean[f]) / sqrt(state.running_var[f] + config.epsilon);
+ output[f] = gamma[f] * x_hat + beta[f];
+ f = f + 1;
+ }
+ }
+
+ // Inference backward (mean and variance are constants here):
+ // grad_input[f] = grad_output[f] gamma[f] / sqrt(var[f] + eps)
+ // grad_gamma[f] = grad_output[f] x_hat[f], grad_beta[f] = grad_output[f]
+ fn backward(grad_output: []const f32, input: []const f32, gamma: []const f32, beta: []const f32, mean: []const f32, var: []const f32, grad_gamma: []f32, grad_beta: []f32, grad_input: []f32, config: BatchNormConfig) -> void {
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const inv_std = 1.0 / sqrt(var[f] + config.epsilon);
+ grad_gamma[f] = grad_output[f] * (input[f] - mean[f]) * inv_std;
+ grad_beta[f] = grad_output[f];
+ grad_input[f] = grad_output[f] * gamma[f] * inv_std;
+ f = f + 1;
+ }
+ }
+
+ // Training: input is batch x num_features, row-major. Each feature is normalised by
+ // its own batch mean and (biased) batch variance, then scaled and shifted.
+ fn forward_train(input: []const f32, batch: usize, gamma: []const f32, beta: []const f32, output: []f32, config: BatchNormConfig) -> void {
+ const nf : usize = config.num_features;
+ var f : usize = 0;
+ while (f < nf) {
+ var mean : f32 = 0.0;
+ var b : usize = 0;
+ while (b < batch) {
+ mean = mean + input[b * nf + f];
+ b = b + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(batch));
+ var variance : f32 = 0.0;
+ b = 0;
+ while (b < batch) {
+ const d = input[b * nf + f] - mean;
+ variance = variance + d * d;
+ b = b + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(batch));
+ b = 0;
+ while (b < batch) {
+ output[b * nf + f] = gamma[f] * (input[b * nf + f] - mean) / sqrt(variance + config.epsilon) + beta[f];
+ b = b + 1;
+ }
+ f = f + 1;
+ }
+ }
+
+ fn cfg() -> BatchNormConfig {
+ return BatchNormConfig{ .num_features = 3, .epsilon = DEFAULT_EPSILON, .momentum = DEFAULT_MOMENTUM, .affine = true, .track_running_stats = true };
+ }
+
+ // forward on input (1, 2, 3), gamma g, beta 0.25, running mean 0 / var 1 (or mean 2 / var 4)
+ fn forward_at(g: f32, shifted: bool, index: usize) -> f32 {
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [g, g, g];
+ const beta : [3]f32 = [0.25, 0.25, 0.25];
+ var mean : [3]f32 = [0.0, 0.0, 0.0];
+ var variance : [3]f32 = [1.0, 1.0, 1.0];
+ if (shifted) {
+ mean = [2.0, 2.0, 2.0];
+ variance = [4.0, 4.0, 4.0];
+ }
+ const state = BatchNormState{ .running_mean = &mean, .running_var = &variance };
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&input, &gamma, &beta, state, &output, cfg());
+ return output[index];
+ }
+
+ // which: 0 = grad_gamma, 1 = grad_beta, 2 = grad_input; grad_output 1, gamma 2, mean 0, var 1
+ fn backward_at(which: u8, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [2.0, 2.0, 2.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ const mean : [3]f32 = [0.0, 0.0, 0.0];
+ const variance : [3]f32 = [1.0, 1.0, 1.0];
+ var grad_gamma : [3]f32 = [9.0] ** 3;
+ var grad_beta : [3]f32 = [9.0] ** 3;
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &gamma, &beta, &mean, &variance, &grad_gamma, &grad_beta, &grad_input, cfg());
+ if (which == 0) {
+ return grad_gamma[index];
+ }
+ if (which == 1) {
+ return grad_beta[index];
+ }
+ return grad_input[index];
+ }
+
+ // forward_train on a batch of 4 samples x 3 features; returns output[b][f]
+ fn train_at(b: usize, f: usize) -> f32 {
+ const input : [12]f32 = [1.0, 10.0, -1.0, 2.0, 20.0, -1.0, 3.0, 30.0, 1.0, 4.0, 40.0, 1.0];
+ const gamma : [3]f32 = [1.0, 1.0, 1.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ var output : [12]f32 = [0.0] ** 12;
+ forward_train(&input, 4, &gamma, &beta, &output, cfg());
+ return output[b * 3 + f];
+ }
+
+ fn valid_input() -> u32 {
+ return cfg().num_features;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // With mean 0, var 1, gamma 1, beta 0 the layer is the identity up to eps:
+ // 1 / sqrt(1 + 1e-5) = 0.999995 (the old `==` held only for the copy placeholder)
+ test "forward_basic_case"
+ given out0 = forward_at(1.0, false, 0) - 0.25
+ when out2 = forward_at(1.0, false, 2) - 0.25
+ then @abs(out0 - 1.0) < 0.00001 and @abs(out2 - 3.0) < 0.00003
+
+ // mean 2, var 4: (x - 2) / 2 * gamma + beta, so x = 3 with gamma 2 gives 1.25
+ test "forward_uses_running_statistics"
+ given result = forward_at(2.0, true, 2)
+ when centre = forward_at(2.0, true, 1)
+ then @abs(result - 1.25) < 0.00001 and centre == 0.25
+
+ // grad_gamma = g x_hat = (1, 2, 3) / sqrt(1 + eps), grad_beta = g = 1,
+ // grad_input = g gamma / sqrt(1 + eps) = 2 (the old 0, 0 were the placeholder's)
+ test "backward_basic_case"
+ given grad_gamma = backward_at(0, 2)
+ when grad_input = backward_at(2, 0)
+ then @abs(grad_gamma - 3.0) < 0.0001 and backward_at(1, 0) == 1.0 and grad_input != 0.0 and @abs(grad_input - 2.0) < 0.0001
+
+ // Each feature of the batch comes out with mean 0 and variance 1
+ test "forward_train_normalises_each_feature"
+ given mean_f1 = train_at(0, 1) + train_at(1, 1) + train_at(2, 1) + train_at(3, 1)
+ when var_f1 = (train_at(0, 1) * train_at(0, 1) + train_at(1, 1) * train_at(1, 1) + train_at(2, 1) * train_at(2, 1) + train_at(3, 1) * train_at(3, 1)) / 4.0
+ then @abs(mean_f1) < 0.00001 and @abs(var_f1 - 1.0) < 0.0001 and @abs(train_at(0, 0) - train_at(0, 1)) < 0.00001 and @abs(train_at(0, 2) + 1.0) < 0.0001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // num_features > 0
+ invariant batchnorm_constraint_0
+ assert valid_input() > 0
+
+ // epsilon > 0
+ invariant batchnorm_constraint_1
+ assert DEFAULT_EPSILON > 0.0
+
+ // 0 <= momentum < 1
+ invariant batchnorm_constraint_2
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+
+ // gamma.size() == num_features: the fixtures carry 3 of each
+ invariant batchnorm_constraint_3
+ assert valid_input() == 3
+
+ // beta.size() == num_features: beta shifts every feature, so x = mean gives exactly beta
+ invariant batchnorm_constraint_4
+ assert forward_at(2.0, true, 1) == 0.25
+
diff --git a/apps/website/public/ru/learn/cross-entropy/card.png b/apps/website/public/ru/learn/cross-entropy/card.png
new file mode 100644
index 0000000000..3e6845c300
Binary files /dev/null and b/apps/website/public/ru/learn/cross-entropy/card.png differ
diff --git a/apps/website/public/ru/learn/cross-entropy/index.html b/apps/website/public/ru/learn/cross-entropy/index.html
new file mode 100644
index 0000000000..c769dd6cba
--- /dev/null
+++ b/apps/website/public/ru/learn/cross-entropy/index.html
@@ -0,0 +1,289 @@
+
+
+
+
+
+
+Перекрёстная энтропия: курс о нейросетях, урок 14 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
Перекрёстная энтропия
+
Вы узнаете
Почему перекрёстная энтропия так дорого берёт за уверенный неверный ответ и почему именно это учит классификатор.
+
Классификатор выдаёт вероятность, а перекрёстная энтропия берёт -log вероятности, которую он дал верному ответу. Предсказания 0.8, 0.3 и 0.9 против ответов 1, 0 и 1 стоят в среднем 0.2284. Быть уверенным и правым почти бесплатно: 0.99, 0.01 и 0.99 стоят 0.0101. Быть уверенным и неправым дорого: 0.01, 0.99 и 0.01 стоят 4.6052, а к 0 стоимость растёт без предела, поэтому спека обрезает p до [1e-7, 1 - 1e-7]. Её градиент наибольший как раз там, где модель уверена и неправа. Спека binary_crossentropy_loss.t27 вычисляет потерю и градиент, из вероятностей или из логитов.
+
Попробовать
Тяните предсказание 0 к 0.01 и смотрите на кривую; затем прочитайте столбики градиента при 0.5, 0.5 и 0.5.
Set three predicted probabilities against the answers 1, 0, 1: the loss is -log of the chance given to the truth. Drag toward a confident mistake and the cost climbs without bound.
+
specs/ml/loss/binary_crossentropy_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Gradient w.r.t predictions | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: never compiled (`new f32[n]`, a float loop index, `predictions.size`,
+// an empty enum), and it approximated sigmoid and log with polynomials that
+// were 0.3 off at x = 1. The probability gradient also had the wrong sign on its
+// second term: d/dp of -(t log p + (1-t) log(1-p)) is -t/p + (1-t)/(1-p). It now
+// uses exp and log, writes gradients into a caller's buffer, and reduces by mean.
+
+module BinaryCe;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_EPSILON : f32 = 1e-7;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const BinaryCEConfig = struct {
+ from_logits : bool,
+ epsilon : f32,
+ reduction : Reduction,
+ };
+
+ pub const Reduction = enum(u8) {
+ Mean,
+ Sum,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn sigmoid(x: f32) -> f32 {
+ if (x >= 0.0) {
+ return 1.0 / (1.0 + exp(-x));
+ }
+ const e = exp(x);
+ return e / (1.0 + e);
+ }
+
+ // p clipped into [eps, 1 - eps] so log stays finite
+ fn clip(p: f32, eps: f32) -> f32 {
+ if (p < eps) {
+ return eps;
+ }
+ if (p > 1.0 - eps) {
+ return 1.0 - eps;
+ }
+ return p;
+ }
+
+ // -(t log p + (1 - t) log(1 - p)) for one element
+ fn element_loss(pred: f32, target: f32, config: BinaryCEConfig) -> f32 {
+ var p = pred;
+ if (config.from_logits) {
+ p = sigmoid(pred);
+ }
+ p = clip(p, config.epsilon);
+ return -(target * log(p) + (1.0 - target) * log(1.0 - p));
+ }
+
+ // mean (or sum) of element_loss
+ fn forward(predictions: []const f32, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ loss = loss + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ if (config.reduction == Reduction.Sum) {
+ return loss;
+ }
+ return loss / @as(f32, @floatFromInt(predictions.len));
+ }
+
+ // A batch of rows of width `width`, flattened row-major: the mean over every element
+ fn forward_batch(predictions: []const f32, width: usize, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ total = total + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ _ = width;
+ return total / @as(f32, @floatFromInt(predictions.len));
+ }
+
+ // Per-element gradient of the (unreduced) loss:
+ // from logits: sigmoid(z) - t
+ // from probabilities: -t / p + (1 - t) / (1 - p)
+ fn gradient(predictions: []const f32, targets: []const f32, config: BinaryCEConfig, grads: []f32) -> void {
+ var i : usize = 0;
+ while (i < predictions.len) {
+ const t = targets[i];
+ if (config.from_logits) {
+ grads[i] = sigmoid(predictions[i]) - t;
+ } else {
+ const p = clip(predictions[i], config.epsilon);
+ grads[i] = -t / p + (1.0 - t) / (1.0 - p);
+ }
+ i = i + 1;
+ }
+ }
+
+ fn cfg(from_logits: bool) -> BinaryCEConfig {
+ return BinaryCEConfig{ .from_logits = from_logits, .epsilon = DEFAULT_EPSILON, .reduction = Reduction.Mean };
+ }
+
+ // The fixture: predictions (a, b, c) against targets (1, 0, 1)
+ fn loss_of(a: f32, b: f32, c: f32, from_logits: bool) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ return forward(&predictions, &targets, cfg(from_logits));
+ }
+
+ fn grad_of(a: f32, b: f32, c: f32, from_logits: bool, index: usize) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ var grads : [3]f32 = [9.0] ** 3;
+ gradient(&predictions, &targets, cfg(from_logits), &grads);
+ return grads[index];
+ }
+
+ fn batch_loss() -> f32 {
+ const predictions : [4]f32 = [0.8, 0.3, 0.9, 0.1];
+ const targets : [4]f32 = [1.0, 0.0, 1.0, 0.0];
+ return forward_batch(&predictions, 2, &targets, cfg(false));
+ }
+
+ fn valid_input() -> f32 {
+ return loss_of(0.8, 0.3, 0.9, false);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test sigmoid_basic_case
+ given input = 0.0
+ when result = sigmoid(input)
+ then @abs(result - 0.5) < 1e-6
+
+ test sigmoid_positive_input
+ given input = 1.0
+ when result = sigmoid(input)
+ then result > 0.5 and result < 1.0 and @abs(result - 0.7310586) < 0.000001
+
+ test sigmoid_negative_input
+ given input = -1.0
+ when result = sigmoid(input)
+ then result > 0.0 and result < 0.5 and @abs(result + sigmoid(1.0) - 1.0) < 0.000001
+
+ // -(ln 0.8 + ln 0.7 + ln 0.9) / 3 = 0.2283930
+ test forward_basic_case
+ given result = loss_of(0.8, 0.3, 0.9, false)
+ then result > 0.0 and @abs(result - 0.2283930) < 0.000001
+
+ // From logits (2, -1, 3) it is the same loss as from their sigmoids
+ test forward_from_logits_case
+ given result = loss_of(2.0, -1.0, 3.0, true)
+ when probs = loss_of(sigmoid(2.0), sigmoid(-1.0), sigmoid(3.0), false)
+ then result > 0.0 and @abs(result - probs) < 0.000001
+
+ // (-ln 0.8 - ln 0.7 - ln 0.9 - ln 0.9) / 4 = 0.1976349
+ test forward_batch_basic_case
+ given result = batch_loss()
+ then result > 0.0 and @abs(result - 0.1976349) < 0.000001
+
+ // t = 1 pulls p up (negative gradient), t = 0 pushes it down (positive)
+ test gradient_basic_case
+ given g0 = grad_of(0.8, 0.3, 0.9, false, 0)
+ when g1 = grad_of(0.8, 0.3, 0.9, false, 1)
+ then g0 < 0.0 and g1 > 0.0 and grad_of(0.8, 0.3, 0.9, false, 2) < 0.0 and @abs(g0 + 1.25) < 0.000001 and @abs(g1 - 1.0 / 0.7) < 0.000001
+
+ // From logits: sigmoid(z) - t, and it matches a finite difference of the loss
+ test gradient_from_logits_case
+ given g1 = grad_of(2.0, -1.0, 3.0, true, 1)
+ when slope = 3.0 * (loss_of(2.0, -1.0 + 0.001, 3.0, true) - loss_of(2.0, -1.0 - 0.001, 3.0, true)) / 0.002
+ then @abs(g1 - sigmoid(-1.0)) < 0.000001 and @abs(g1 - slope) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // targets[i] in {0, 1}: a perfect prediction costs (almost) nothing
+ invariant binary_ce_constraint_0
+ assert loss_of(1.0, 0.0, 1.0, false) < 0.000001
+
+ // 0 < epsilon << 1
+ invariant binary_ce_constraint_1
+ assert DEFAULT_EPSILON > 0.0 and DEFAULT_EPSILON < 0.001
+
+ // name: binary_ce -- positive for an imperfect prediction
+ invariant binary_ce_constraint_2
+ assert valid_input() > 0.0
+
+ // name: gradient -- zero where the logit already gives the target
+ invariant binary_ce_constraint_3
+ assert @abs(grad_of(30.0, -30.0, 30.0, true, 0)) < 0.000001
+
+ // "Bishop (2006) - Pattern Recognition and Machine Learning": the loss falls as p nears t
+ invariant binary_ce_constraint_4
+ assert loss_of(0.9, 0.1, 0.9, false) < loss_of(0.6, 0.4, 0.6, false)
+
+ // "Goodfellow et al. (2016) - Deep Learning": clipping keeps a certain wrong answer finite
+ invariant binary_ce_constraint_5
+ assert loss_of(0.0, 1.0, 0.0, false) < 20.0
+
Почему выключение нейронов во время обучения помогает и почему выживших нужно масштабировать.
+
Во время обучения dropout случайно выключает долю p нейронов, чтобы остальные не опирались ни на один из них. Выжившие нейроны тогда нужно масштабировать, иначе следующий слой видел бы сигнал другого размера при обучении, чем при выводе. Обратный dropout умножает их на 1 / (1 - p) при обучении: с p = 0.5 вход 1, 2, 3, 4 становится 2, 0, 6, 0, а вывод это простой проход. Классический dropout оставляет 1, 0, 3, 0 и умножает на 0.5 уже при выводе. Спека dropout_layer.t27 использует фиксированную маску, оставить, выбросить, оставить, выбросить, поэтому числа каждый раз одинаковые.
+
Попробовать
Сравните выходы четырёх режимов; затем прочитайте градиент выброшенного нейрона при обучении.
Switch between training and inference, inverted and classic dropout: units 1 and 3 are dropped, the survivors are scaled so that the layer's output keeps its expected size.
+
specs/ml/layers/dropout_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Prevent co-adaptation of neurons | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the tests called `allocf32` and `allocbool`, which no spec declares, and
+// asserted `result != undefined`. Inference also scaled by (1 - p) even for
+// inverted dropout, which already scaled by 1 / (1 - p) in training, so the
+// expected activation shrank by (1 - p) between training and inference. Inference
+// is now the identity when scale_during_training is set.
+
+module Dropout;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_DROP_RATE : f32 = 0.5;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const DropoutConfig = struct {
+ p : f32,
+ inplace : bool,
+ scale_during_training : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(input: []const f32, output: []f32, mask: []bool, training: bool, config: DropoutConfig) → void
+ fn forward(input: []const f32, output: []f32, mask: []bool, training: bool, config: DropoutConfig) -> void {
+ const n = input.len;
+ var i : usize = 0;
+ while (i < n) {
+ if (training) {
+ if (mask[i]) {
+ // Neuron is kept, scale during training if configured
+ if (config.scale_during_training) {
+ output[i] = input[i] * (1.0 / (1.0 - config.p));
+ } else {
+ output[i] = input[i];
+ }
+ } else {
+ // Neuron is dropped
+ output[i] = 0.0;
+ }
+ } else {
+ if (config.scale_during_training) {
+ // Inverted dropout already scaled in training: inference is the identity
+ output[i] = input[i];
+ } else {
+ // Classic dropout: scale by the keep probability at inference
+ output[i] = input[i] * (1.0 - config.p);
+ }
+ }
+ i = i + 1;
+ }
+ }
+
+ // backward(grad_output: []const f32, mask: []const bool, grad_input: []f32, config: DropoutConfig) → void
+ fn backward(grad_output: []const f32, mask: []const bool, grad_input: []f32, config: DropoutConfig) -> void {
+ const n = grad_output.len;
+ var i : usize = 0;
+ while (i < n) {
+ if (mask[i]) {
+ // Gradient flows through kept neurons
+ if (config.scale_during_training) {
+ grad_input[i] = grad_output[i] * (1.0 / (1.0 - config.p));
+ } else {
+ grad_input[i] = grad_output[i];
+ }
+ } else {
+ // Gradient is zero for dropped neurons
+ grad_input[i] = 0.0;
+ }
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ fn cfg(scaled: bool) -> DropoutConfig {
+ return DropoutConfig{ .p = DEFAULT_DROP_RATE, .inplace = false, .scale_during_training = scaled };
+ }
+
+ // forward on input (1, 2, 3, 4) with mask (keep, drop, keep, drop)
+ fn forward_at(training: bool, scaled: bool, index: usize) -> f32 {
+ const input : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [4]f32 = [9.0] ** 4;
+ var mask : [4]bool = [true, false, true, false];
+ forward(&input, &output, &mask, training, cfg(scaled));
+ return output[index];
+ }
+
+ fn backward_at(scaled: bool, index: usize) -> f32 {
+ const grad_output : [4]f32 = [1.0, 1.0, 1.0, 1.0];
+ const mask : [4]bool = [true, false, true, false];
+ var grad_input : [4]f32 = [9.0] ** 4;
+ backward(&grad_output, &mask, &grad_input, cfg(scaled));
+ return grad_input[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // p = 0.5, inverted: kept units are doubled, dropped ones are 0
+ test forward_basic_case
+ given kept = forward_at(true, true, 2)
+ when dropped = forward_at(true, true, 1)
+ then kept == 6.0 and dropped == 0.0 and forward_at(true, true, 0) == 2.0
+
+ // The expected activation is the same in training and inference
+ test forward_inference_keeps_the_expectation
+ given inverted = forward_at(false, true, 2)
+ when classic = forward_at(false, false, 2)
+ then inverted == 3.0 and classic == 1.5 and forward_at(true, false, 2) == 3.0
+
+ // The gradient flows only through kept units, with the same scale
+ test backward_basic_case
+ given kept = backward_at(true, 0)
+ when dropped = backward_at(true, 3)
+ then kept == 2.0 and dropped == 0.0 and backward_at(false, 2) == 1.0
+
+ invariant dropout_rate_is_a_probability
+ assert DEFAULT_DROP_RATE >= 0.0 and DEFAULT_DROP_RATE < 1.0
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/elu/card.png b/apps/website/public/ru/learn/elu/card.png
new file mode 100644
index 0000000000..a0ac7b7a3c
Binary files /dev/null and b/apps/website/public/ru/learn/elu/card.png differ
diff --git a/apps/website/public/ru/learn/elu/index.html b/apps/website/public/ru/learn/elu/index.html
new file mode 100644
index 0000000000..a49d27db61
--- /dev/null
+++ b/apps/website/public/ru/learn/elu/index.html
@@ -0,0 +1,197 @@
+
+
+
+
+
+
+ELU: пол, а не обрыв: курс о нейросетях, урок 6 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
ELU: пол, а не обрыв
+
Вы узнаете
Как ELU сохраняет мягкую отрицательную сторону с полом на -alpha и почему её наклон никогда не умирает до конца.
+
ELU сохраняет положительную сторону ReLU и заменяет обрыв на отрицательной стороне кривой alpha (e^x - 1), которая выполаживается к -alpha. При alpha 1 вход x = -3 даёт -0.9502, уже близко к полу -1, а наклон там 0.0498: мал, но не 0, так что нейрон ещё может сдвинуться. Отрицательные выходы не выбрасываются, поэтому среднее выходов слоя ближе к 0. Спека elu_activation.t27 вычисляет кривую, наклон и пол, а также свой пакет при -1, 0 и 2: -0.632, 0 и 2.
+
Попробовать
Поставьте alpha на 0.5 и прочитайте пол; затем найдите, как далеко влево должен уйти x, чтобы наклон стал меньше 0.01.
Drag along ELU: x above zero, alpha (e^x - 1) below. The negative side never falls under -alpha, so outputs average closer to zero, and its slope never quite dies.
+
specs/ml/activation/elu_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Derivative | φ² + 1/φ² = 3 | TRINITY
+// #8980: `math::exp` named a module this spec never imports, forward_batch grew
+// an array with `+`, and every test called a two-argument function with one
+// argument and asserted `result != undefined`. Now each test checks a value.
+
+module Elu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_ALPHA : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const ELUConfig = struct {
+ alpha : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(x, alpha) = x for x > 0, alpha * (exp(x) - 1) otherwise
+ fn forward(x: f32, alpha: f32) -> f32 {
+ if (x > 0.0) {
+ return x;
+ }
+ return alpha * (exp(x) - 1.0);
+ }
+
+ // forward_batch: output[i] = forward(input[i], alpha)
+ fn forward_batch(input: []const f32, alpha: f32, output: []f32) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = forward(input[i], alpha);
+ i = i + 1;
+ }
+ }
+
+ // derivative(x, alpha) = 1 for x > 0, alpha * exp(x) otherwise
+ fn derivative(x: f32, alpha: f32) -> f32 {
+ if (x > 0.0) {
+ return 1.0;
+ }
+ return alpha * exp(x);
+ }
+
+ // The inputs the tests and invariants share: one negative, zero, one positive.
+ fn negative_input() -> f32 {
+ return -1.0;
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
+ }
+
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [-1.0, 0.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, DEFAULT_ALPHA, &output);
+ return output[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // ELU(-1) = exp(-1) - 1 = -0.6321206; ELU(2) = 2
+ test "forward_basic_case"
+ given input = negative_input()
+ when result = forward(input, DEFAULT_ALPHA)
+ then @abs(result - (-0.6321206)) < 0.000001 and forward(valid_input(), DEFAULT_ALPHA) == 2.0
+
+ // alpha scales only the negative side, and ELU(x) never goes below -alpha (in f32
+ // ELU(-20) rounds to exactly -alpha)
+ test "forward_alpha_scales_negative_side"
+ given input = negative_input()
+ when result = forward(input, 0.5)
+ then @abs(result - 0.5 * forward(input, 1.0)) < 0.000001 and forward(-20.0, 0.5) >= -0.5
+
+ test "forward_batch_basic_case"
+ given input = negative_input()
+ when result = batch_at(0)
+ then result == forward(input, DEFAULT_ALPHA) and batch_at(1) == 0.0 and batch_at(2) == 2.0
+
+ // ELU'(-1) = exp(-1) = 0.3678794; ELU'(2) = 1
+ test "derivative_basic_case"
+ given input = negative_input()
+ when result = derivative(input, DEFAULT_ALPHA)
+ then @abs(result - 0.3678794) < 0.000001 and derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+
+ test "derivative_matches_finite_difference"
+ given slope = (forward(-1.0 + 0.001, DEFAULT_ALPHA) - forward(-1.0 - 0.001, DEFAULT_ALPHA)) / 0.002
+ when result = derivative(-1.0, DEFAULT_ALPHA)
+ then @abs(result - slope) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ invariant elu_constraint_0
+ assert DEFAULT_ALPHA > 0.0
+
+ // name: elu_formula -- identity on the positive side
+ invariant elu_constraint_1
+ assert forward(valid_input(), DEFAULT_ALPHA) == valid_input()
+
+ // name: derivative -- 1 on the positive side
+ invariant elu_constraint_2
+ assert derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+
+ // "Clevert et al. (2016) - Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)":
+ // ELU is continuous at 0
+ invariant elu_constraint_3
+ assert forward(0.0, DEFAULT_ALPHA) == 0.0
+
Как токен превращается в строку чисел и почему обучение меняет только прочитанные строки.
+
Сеть не умеет умножать слово, поэтому каждый токен это индекс в таблице с одной строкой чисел на токен, а поиск токена копирует его строку. В спеке в таблице 4 токена шириной 2, и строка v хранит (10 v, 10 v + 1), так что токен 2 читает (20, 21). Обучение отправляет градиент назад только в прочитанные строки, а токен, прочитанный дважды, собирает оба: строки градиента (1, 2) и (3, 4) на токене 2 складываются в (4, 6). Спека embedding_layer.t27 вычисляет поиск и градиент.
+
Попробовать
Прочитайте токен 2 дважды, а затем токены 1 и 3; каждый раз сосчитайте строки, где градиент не 0.
Pick two tokens: each one copies its row out of the table. Then the gradient flows back only into the rows that were read, and a token read twice collects both.
+
specs/ml/layers/embedding_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Embeddings are learned parameters | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward only asserted its arguments and wrote nothing, the tests called
+// undeclared `test_input_forward` / `test_input_init` and asserted
+// `result != undefined`, and the invariants asserted `true`. forward now looks each
+// token's row up in W (vocab_size x d_model, row-major), backward adds each
+// gradient row back into the row it came from, and init fills W deterministically.
+
+module Embedding;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_D_MODEL : u32 = 64;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const EmbeddingConfig = struct {
+ vocab_size : u32,
+ d_model : u32,
+ };
+
+ pub const EmbeddingWeights = struct {
+ W : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // embeddings[t * d + j] = W[token_ids[t] * d + j]: one row of W per token
+ fn forward(token_ids: []const u32, weights: EmbeddingWeights, embeddings: []f32, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ embeddings[t * d + j] = weights.W[row * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
+ }
+ }
+
+ // grad_W[token_ids[t]] += grad_embeddings[t]: a token seen twice collects both rows
+ fn backward(token_ids: []const u32, grad_embeddings: []const f32, grad_W: []f32, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ grad_W[row * d + j] = grad_W[row * d + j] + grad_embeddings[t * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
+ }
+ }
+
+ // Deterministic init in [-0.5, 0.5) / sqrt(d_model): a fixed hash of (row, column), so
+ // two runs start from the same weights
+ fn init(weights: EmbeddingWeights, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d)));
+ var k : usize = 0;
+ while (k < config.vocab_size * d) {
+ const h : u32 = @as(u32, @intCast((k * 2654435761) % 1000));
+ weights.W[k] = (@as(f32, @floatFromInt(h)) / 1000.0 - 0.5) * scale;
+ k = k + 1;
+ }
+ }
+
+ // vocab 4, d_model 2, W row v = (10 v, 10 v + 1)
+ fn cfg() -> EmbeddingConfig {
+ return EmbeddingConfig{ .vocab_size = 4, .d_model = 2 };
+ }
+
+ fn lookup_at(a: u32, b: u32, index: usize) -> f32 {
+ var w : [8]f32 = [0.0, 1.0, 10.0, 11.0, 20.0, 21.0, 30.0, 31.0];
+ const ids : [2]u32 = [a, b];
+ var out : [4]f32 = [-1.0] ** 4;
+ forward(&ids, EmbeddingWeights{ .W = &w }, &out, cfg());
+ return out[index];
+ }
+
+ // grad_W[index] after backward of ids (a, b) with gradient rows (1, 2) and (3, 4)
+ fn grad_at(a: u32, b: u32, index: usize) -> f32 {
+ const ids : [2]u32 = [a, b];
+ const grads : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_w : [8]f32 = [0.0] ** 8;
+ backward(&ids, &grads, &grad_w, cfg());
+ return grad_w[index];
+ }
+
+ fn init_at(index: usize) -> f32 {
+ var w : [8]f32 = [9.0] ** 8;
+ init(EmbeddingWeights{ .W = &w }, cfg());
+ return w[index];
+ }
+
+ fn valid_input() -> u32 {
+ return cfg().vocab_size * cfg().d_model;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // ids (2, 0): rows (20, 21) then (0, 1)
+ test forward_basic_case
+ given first = lookup_at(2, 0, 0)
+ when last = lookup_at(2, 0, 3)
+ then first == 20.0 and lookup_at(2, 0, 1) == 21.0 and lookup_at(2, 0, 2) == 0.0 and last == 1.0
+
+ // Every weight is written, inside [-0.5, 0.5) / sqrt(2), and init is reproducible
+ test init_basic_case
+ given first = init_at(0)
+ when last = init_at(7)
+ then first != 9.0 and last != 9.0 and @abs(first) <= 0.36 and @abs(last) <= 0.36 and init_at(5) == init_at(5) and init_at(1) != init_at(2)
+
+ // A token seen twice collects both gradient rows; an unseen row stays 0
+ test backward_accumulates_repeated_tokens
+ given row1 = grad_at(1, 1, 2)
+ when unseen = grad_at(1, 1, 0)
+ then row1 == 4.0 and grad_at(1, 1, 3) == 6.0 and unseen == 0.0 and grad_at(1, 3, 6) == 3.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // vocab_size > 0
+ invariant embedding_constraint_0
+ assert cfg().vocab_size > 0
+
+ // d_model > 0
+ invariant embedding_constraint_1
+ assert cfg().d_model > 0 and DEFAULT_D_MODEL > 0
+
+ // W.size() == vocab_size * d_model
+ invariant embedding_constraint_2
+ assert valid_input() == 8
+
+ // token_ids must be in [0, vocab_size): the last id reads the last row
+ invariant embedding_constraint_3
+ assert lookup_at(3, 3, 1) == 31.0
+
+ // "Embedding dimension affects model capacity": each token gets d_model numbers
+ invariant embedding_constraint_4
+ assert lookup_at(1, 2, 1) == 11.0 and lookup_at(1, 2, 2) == 20.0
+
+ // "Larger vocab requires larger d_model": the init scale shrinks as 1 / sqrt(d_model)
+ invariant embedding_constraint_5
+ assert 1.0 / sqrt(@as(f32, 64.0)) < 1.0 / sqrt(@as(f32, 2.0))
+
+ // "Pre-trained embeddings transfer well": forward is a pure lookup, so loaded rows come out unchanged
+ invariant embedding_constraint_6
+ assert lookup_at(0, 1, 0) == 0.0 and lookup_at(0, 1, 2) == 10.0
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/gelu/card.png b/apps/website/public/ru/learn/gelu/card.png
new file mode 100644
index 0000000000..07239e5293
Binary files /dev/null and b/apps/website/public/ru/learn/gelu/card.png differ
diff --git a/apps/website/public/ru/learn/gelu/index.html b/apps/website/public/ru/learn/gelu/index.html
new file mode 100644
index 0000000000..c153a8ff92
--- /dev/null
+++ b/apps/website/public/ru/learn/gelu/index.html
@@ -0,0 +1,242 @@
+
+
+
+
+
+
+GELU: провал ниже нуля: курс о нейросетях, урок 4 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
GELU: провал ниже нуля
+
Вы узнаете
Что вычисляет GELU, почему она проваливается ниже нуля и насколько близко подходит её приближение через tanh.
+
GELU взвешивает x вероятностью того, что стандартная нормальная величина меньше него: GELU(x) = x Phi(x). Большой положительный вход проходит, большой отрицательный исчезает, а между ними кривая проваливается примерно до -0.17 около x = -0.75 и только потом растёт. BERT и GPT используют её в своих слоях прямого распространения. Точной форме нужна erf, и спека вычисляет её приближением Абрамовица-Стиган с точностью 1.5e-7, а также вычисляет приближение через tanh: при -0.75 они дают -0.16997 и -0.17004. Спека gelu_activation.t27 вычисляет обе кривые и их наклоны.
+
Попробовать
Найдите нижнюю точку кривой по её наклону; затем сравните точную форму и форму через tanh при -3.
Drag along GELU: x weighed by the chance a normal draw is below it. It dips to about -0.17 before it rises, and the tanh shortcut stays within a few thousandths of the exact form.
+
specs/ml/activation/gelu_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Based on cumulative distribution function | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: Zig has neither `@tanh` nor `@erf`, so this never compiled. erf is now
+// a function of this spec (Abramowitz-Stegun 7.1.26, |error| < 1.5e-7), and the
+// exact derivative divides the normal density by sqrt(2 pi), not sqrt(2) (the
+// slip was invisible at x = 0, the only point the tests checked).
+
+module Gelu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const SQRT_2_OVER_PI : f32 = 0.7978845608;
+ const SQRT_PI_OVER_2 : f32 = 1.2533141373;
+ const SQRT_2 : f32 = 1.4142135624;
+ const SQRT_2_PI : f32 = 2.5066282746;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const GELUConfig = struct {
+ approximate : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // erf(x), Abramowitz-Stegun 7.1.26: max absolute error 1.5e-7
+ fn erf(x: f32) -> f32 {
+ if (x < 0.0) {
+ return -erf(-x);
+ }
+ const t = 1.0 / (1.0 + 0.3275911 * x);
+ const poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))));
+ return 1.0 - poly * exp(-x * x);
+ }
+
+ // One element. approximate: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3)));
+ // exact: 0.5 x (1 + erf(x / sqrt(2))) = x Phi(x)
+ fn gelu_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ return 0.5 * x * (1.0 + tanh(inner));
+ }
+ return 0.5 * x * (1.0 + erf(x / SQRT_2));
+ }
+
+ // d/dx of gelu_one
+ fn gelu_grad_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ const tanh_val = tanh(inner);
+ const sech_sq = 1.0 - tanh_val * tanh_val;
+ return 0.5 * (1.0 + tanh_val) + 0.5 * x * sech_sq * SQRT_2_OVER_PI * (1.0 + 3.0 * 0.044715 * x * x);
+ }
+ // Phi(x) + x phi(x), phi(x) = exp(-x^2 / 2) / sqrt(2 pi)
+ const cdf = 0.5 * (1.0 + erf(x / SQRT_2));
+ const pdf = exp(-0.5 * x * x) / SQRT_2_PI;
+ return cdf + x * pdf;
+ }
+
+ fn forward(input: []const f32, output: []f32, config: GELUConfig) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = gelu_one(input[i], config.approximate);
+ i = i + 1;
+ }
+ }
+
+ fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: GELUConfig) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ grad_input[i] = grad_output[i] * gelu_grad_one(input[i], config.approximate);
+ i = i + 1;
+ }
+ }
+
+ // forward / backward over a one-element slice, for the tests
+ fn forward_at(x: f32, approximate: bool) -> f32 {
+ const input : [1]f32 = [x];
+ var output : [1]f32 = [0.0];
+ forward(&input, &output, GELUConfig{ .approximate = approximate });
+ return output[0];
+ }
+
+ fn backward_at(x: f32, approximate: bool) -> f32 {
+ const grad_output : [1]f32 = [1.0];
+ const input : [1]f32 = [x];
+ var grad_input : [1]f32 = [0.0];
+ backward(&grad_output, &input, &grad_input, GELUConfig{ .approximate = approximate });
+ return grad_input[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // The product goes through a function so the check runs, not only folds at compile time.
+ fn product_of(a: f32, b: f32) -> f32 {
+ return a * b;
+ }
+
+ fn config_of(approximate: bool) -> GELUConfig {
+ return GELUConfig{ .approximate = approximate };
+ }
+
+ test gelu_sqrt_constants_are_reciprocals
+ given product = product_of(SQRT_2_OVER_PI, SQRT_PI_OVER_2)
+ then @abs(product - 1.0) < 0.0000005
+
+ test gelu_sqrt_2_over_pi_squares_to_2_over_pi
+ given squared = product_of(SQRT_2_OVER_PI, SQRT_2_OVER_PI)
+ then @abs(squared - 0.63661977) < 0.0000003
+
+ test gelu_config_records_the_approximation_choice
+ given exact = config_of(false)
+ when approx = config_of(true)
+ then !exact.approximate and approx.approximate
+
+ // erf(0) = 0, erf(1) = 0.8427008, erf(-1) = -erf(1)
+ test gelu_erf_known_values
+ given at_one = erf(1.0)
+ when at_zero = erf(0.0)
+ then @abs(at_zero) < 0.0000002 and @abs(at_one - 0.8427008) < 0.0000003 and erf(-1.0) == -at_one
+
+ test gelu_forward_zero_input
+ given exact = forward_at(0.0, false)
+ when approx = forward_at(0.0, true)
+ then exact == 0.0 and approx == 0.0
+
+ test gelu_backward_zero_input_derivative
+ given exact = backward_at(0.0, false)
+ when approx = backward_at(0.0, true)
+ then @abs(exact - 0.5) < 0.0000002 and approx == 0.5
+
+ test gelu_forward_monotonic_positive_tail
+ given small = forward_at(1.0, true)
+ when large = forward_at(2.0, true)
+ then small < large
+
+ // With the cubic term, approximate GELU(1) = 0.8411920; exact GELU(1) = Phi(1) = 0.8413447.
+ // Without it, 0.5 (1 + tanh(0.7978846)) = 0.8296438 -- so this pins the term.
+ test gelu_tanh_cubic_coefficient
+ given approx = forward_at(1.0, true)
+ when exact = forward_at(1.0, false)
+ then @abs(approx - 0.8411920) < 0.000001 and @abs(exact - 0.8413447) < 0.000001
+
+ test gelu_backward_exact_derivative_at_zero
+ given result = backward_at(0.0, false)
+ then @abs(result - 0.5) < 0.0000002
+
+ test gelu_forward_approx_derivative_at_zero
+ given result = backward_at(0.0, true)
+ then result == 0.5
+
+ // The derivative at x = 1 is where the old sqrt(2) slip showed: 1.0833155 exact
+ test gelu_backward_matches_finite_difference
+ given exact_slope = (forward_at(1.001, false) - forward_at(0.999, false)) / 0.002
+ when approx_slope = (forward_at(1.001, true) - forward_at(0.999, true)) / 0.002
+ then @abs(backward_at(1.0, false) - exact_slope) < 0.002 and @abs(backward_at(1.0, true) - approx_slope) < 0.002 and @abs(backward_at(1.0, false) - 1.0833155) < 0.000002
+
Как градиентный спуск идёт вниз по функции потерь, что добавляет момент и почему слишком большой шаг никогда не успокаивается.
+
Обучение это спуск под гору по функции потерь. Градиентный спуск каждый шаг сдвигает вес на -lr, умноженное на его градиент; в спеке потеря равна (x - 3)^2 от x = 0, и lr 0.1 доходит до 2.99996 за 50 шагов. Момент добавляет долю прошлого сдвига, что ускоряет длинный склон и перелетает острое дно: при 0.9 вес пролетает 3 примерно до 5.13 и, раскачиваясь, приходит к 3.09 после 50 шагов. Нестеров берёт градиент на шаг вперёд и качается меньше, примерно до 4.25. Слишком большой шаг никогда не успокаивается: при lr 1 вес навсегда скачет между 0 и 6. Спека sgd.t27 вычисляет каждый шаг.
+
Попробовать
Найдите наибольший шаг обучения, который ещё успокаивается без момента; затем добавьте момент 0.9 и сосчитайте качания.
Set the learning rate and momentum and watch a weight walk from 0 to the bottom of (x - 3)^2. Plain steps creep, momentum overshoots and rings, and a rate of 1 bounces forever.
+
specs/ml/optimizer/sgd.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Nesterov accelerated gradient (NAG) | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: init called `new f32[n]`; update read undeclared locals; dampening and
+// nesterov were ignored; and weight decay was ADDED to the parameter
+// (`- v + wd * p`), which grows the weights it should shrink. update is now
+// PyTorch's SGD: g = grad + wd p; v = mu v + (1 - dampening) g;
+// step = g + mu v (Nesterov) or v; p = p - lr step.
+
+module Sgd;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_LR : f32 = 0.01;
+ const DEFAULT_MOMENTUM : f32 = 0.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SGDConfig = struct {
+ learning_rate : f32,
+ momentum : f32,
+ weight_decay : f32,
+ dampening : f32,
+ nesterov : bool,
+ };
+
+ pub const SGDState = struct {
+ velocity : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // init: the first param_count velocities start at zero (the caller owns the buffer)
+ fn init(state: SGDState, param_count: u32) -> void {
+ var i : usize = 0;
+ while (i < param_count) {
+ state.velocity[i] = 0.0;
+ i = i + 1;
+ }
+ }
+
+ fn update(params: []f32, grads: []const f32, state: SGDState, config: SGDConfig) -> void {
+ var i : usize = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ var step = g;
+ if (config.momentum > 0.0) {
+ state.velocity[i] = config.momentum * state.velocity[i] + (1.0 - config.dampening) * g;
+ if (config.nesterov) {
+ step = g + config.momentum * state.velocity[i];
+ } else {
+ step = state.velocity[i];
+ }
+ }
+ params[i] = params[i] - config.learning_rate * step;
+ i = i + 1;
+ }
+ }
+
+ fn cfg(lr: f32, momentum: f32, wd: f32, nesterov: bool) -> SGDConfig {
+ return SGDConfig{ .learning_rate = lr, .momentum = momentum, .weight_decay = wd, .dampening = 0.0, .nesterov = nesterov };
+ }
+
+ // Minimise f(x) = (x - 3)^2 (gradient 2 (x - 3)) from x = 0 for `steps` steps
+ fn descend(lr: f32, momentum: f32, nesterov: bool, steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var velocity : [1]f32 = [5.0];
+ const state = SGDState{ .velocity = &velocity };
+ init(state, 1);
+ var n : u32 = 0;
+ while (n < steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ update(¶ms, &grads, state, cfg(lr, momentum, 0.0, nesterov));
+ n = n + 1;
+ }
+ return params[0];
+ }
+
+ fn init_zeroes(index: usize) -> f32 {
+ var velocity : [3]f32 = [7.0, 7.0, 7.0];
+ init(SGDState{ .velocity = &velocity }, 3);
+ return velocity[index];
+ }
+
+ // One step on params (1, 2), grads (0.1, 0.2), momentum 0.9, weight decay 0.001
+ fn one_step(index: usize) -> f32 {
+ var params : [2]f32 = [1.0, 2.0];
+ const grads : [2]f32 = [0.1, 0.2];
+ var velocity : [2]f32 = [0.0, 0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.01, 0.9, 0.001, false));
+ return params[index];
+ }
+
+ // A step with zero gradient and weight decay shrinks the weight toward 0
+ fn decay_step() -> f32 {
+ var params : [1]f32 = [2.0];
+ const grads : [1]f32 = [0.0];
+ var velocity : [1]f32 = [0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.1, 0.0, 0.5, false));
+ return params[0];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_LR;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test "init_basic_case"
+ given first = init_zeroes(0)
+ when last = init_zeroes(2)
+ then first == 0.0 and last == 0.0
+
+ // v = g + wd p (momentum starts at 0): p0 = 1 - 0.01 (0.1 + 0.001) = 0.99899
+ test "update_basic_case"
+ given p0 = one_step(0)
+ when p1 = one_step(1)
+ then p0 != 1.0 and p1 != 2.0 and @abs(p0 - 0.99899) < 0.000001 and @abs(p1 - (2.0 - 0.01 * 0.202)) < 0.000001
+
+ // Weight decay pulls the weight toward 0: 2 - 0.1 * 0.5 * 2 = 1.9
+ test "update_weight_decay_shrinks"
+ given result = decay_step()
+ then @abs(result - 1.9) < 0.000001
+
+ // Plain, momentum and Nesterov SGD all reach the minimum at 3
+ test "update_descends_a_quadratic"
+ given plain = descend(0.1, 0.0, false, 100)
+ when heavy = descend(0.05, 0.9, false, 300)
+ then @abs(plain - 3.0) < 0.0001 and @abs(heavy - 3.0) < 0.0001 and @abs(descend(0.05, 0.9, true, 300) - 3.0) < 0.0001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // learning_rate > 0
+ invariant sgd_constraint_0
+ assert valid_input() > 0.0
+
+ // 0 <= momentum < 1
+ invariant sgd_constraint_1
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+
+ // weight_decay >= 0: a non-negative decay never grows a weight with zero gradient
+ invariant sgd_constraint_2
+ assert decay_step() < 2.0
+
+ // params.size() == grads.size(): every parameter moves
+ invariant sgd_constraint_3
+ assert one_step(0) != 1.0 and one_step(1) != 2.0
+
+ // If momentum > 0: state.velocity.size() == params.size() -- init zeroes all of it
+ invariant sgd_constraint_4
+ assert init_zeroes(1) == 0.0
+
Перед курсом по FPGA: что такое спека t27, как компилятор её читает и что отдаёт в ответ. Каждый урок открывает таблицу или схему из своей спеки и одну спеку, которую можно запустить в браузере.
Курс 0, 27 уроков, в каждом инфографика и спека t27: модуль, константы и типы, функции и тесты, затем стадии компилятора и 7 языков, которые он пишет. Идёт перед курсом по FPGA.
diff --git a/apps/website/public/ru/learn/lamb-and-large-batches/card.png b/apps/website/public/ru/learn/lamb-and-large-batches/card.png
new file mode 100644
index 0000000000..245eb98fdc
Binary files /dev/null and b/apps/website/public/ru/learn/lamb-and-large-batches/card.png differ
diff --git a/apps/website/public/ru/learn/lamb-and-large-batches/index.html b/apps/website/public/ru/learn/lamb-and-large-batches/index.html
new file mode 100644
index 0000000000..3a6803d768
--- /dev/null
+++ b/apps/website/public/ru/learn/lamb-and-large-batches/index.html
@@ -0,0 +1,321 @@
+
+
+
+
+
+
+LAMB и большие пакеты: курс о нейросетях, урок 21 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
LAMB и большие пакеты
+
Вы узнаете
Как LAMB масштабирует шаг каждого слоя его собственной нормой весов, чтобы один шаг обучения подходил слоям любого размера.
+
Обучение на очень больших пакетах требует большого шага обучения, а один шаг может быть слишком большим для малого слоя и слишком малым для большого. LAMB берёт обновление в духе Adam для каждого слоя и перемасштабирует его коэффициентом доверия, нормой весов слоя, ограниченной 10, делённой на норму обновления, поэтому каждый слой сдвигается на одну и ту же долю самого себя. В спеке веса (1, 2), умноженные на масштаб, делают шаг с нормой, в 0.001 раз меньшей их собственной: 0.00447 при масштабе 2, а когда норма весов переходит 10, около масштаба 4.47, шаг остаётся на 0.01. Спека lamb.t27 вычисляет обновление для одного слоя и для двух сразу.
+
Попробовать
Прочитайте шаг при масштабах 1 и 2 и сравните; затем перетащите за изгиб и объясните, почему линия становится плоской.
Scale a layer's weights and watch its LAMB step: it grows with the weight norm, so every layer moves by the same fraction of itself, until the norm passes 10 and the step stops growing.
+
specs/ml/optimizer/lamb.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// LAMB parameter update with trust ratio clipping | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: built with `append` on `[]` and `range` / `len` / `pow`, so it never
+// compiled. The math was also not LAMB: v_hat used beta1's bias correction, the
+// "trust ratio" was ||g|| / ||v_hat|| and was computed, then never applied to the
+// step, and m and v were never written back. It is now You et al. (2020):
+// r = m_hat / (sqrt(v_hat) + eps) + wd * w, trust = min(||w||, clip) / ||r||,
+// w = w - lr * trust * r,
+// per layer, over buffers the caller owns.
+
+module Lamb;
+ use base::types;
+ use math::constants;
+ use ml::optimizer::adam;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_LR : f32 = 0.001;
+ const DEFAULT_BETA1 : f32 = 0.9;
+ const DEFAULT_BETA2 : f32 = 0.999;
+ const DEFAULT_EPSILON : f32 = 1e-6;
+ const DEFAULT_CLIP_THRESHOLD : f32 = 10.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const LAMBConfig = struct {
+ learning_rate : f32,
+ beta1 : f32,
+ beta2 : f32,
+ epsilon : f32,
+ weight_decay : f32,
+ clip_threshold : f32,
+ };
+
+ pub const LAMBState = struct {
+ m : []f32,
+ v : []f32,
+ step : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // m and v are the first num_params entries of the given buffers, zeroed; step 0
+ fn init_state(m: []f32, v: []f32, num_params: u32) -> LAMBState {
+ var i : usize = 0;
+ while (i < num_params) {
+ m[i] = 0.0;
+ v[i] = 0.0;
+ i = i + 1;
+ }
+ return LAMBState{ .m = m[0..num_params], .v = v[0..num_params], .step = 0 };
+ }
+
+ // powi (b^t for a whole t) comes from specs/ml/optimizer/adam.t27 through `use`
+
+ // Writes the moments for step (state.step + 1) into state.m / state.v and the update to
+ // ADD to params into `update`; returns update[0..n]
+ fn compute_layer_update(params: []const f32, grads: []const f32, state: LAMBState, config: LAMBConfig, update: []f32) -> []f32 {
+ const n = params.len;
+ const t = state.step + 1;
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ var w_norm : f32 = 0.0;
+ var r_norm : f32 = 0.0;
+ var i : usize = 0;
+ while (i < n) {
+ state.m[i] = config.beta1 * state.m[i] + (1.0 - config.beta1) * grads[i];
+ state.v[i] = config.beta2 * state.v[i] + (1.0 - config.beta2) * grads[i] * grads[i];
+ const r = (state.m[i] / c1) / (sqrt(state.v[i] / c2) + config.epsilon) + config.weight_decay * params[i];
+ update[i] = r;
+ w_norm = w_norm + params[i] * params[i];
+ r_norm = r_norm + r * r;
+ i = i + 1;
+ }
+ w_norm = sqrt(w_norm);
+ r_norm = sqrt(r_norm);
+ var trust : f32 = 1.0;
+ if (w_norm > 0.0 and r_norm > 0.0) {
+ trust = @min(w_norm, config.clip_threshold) / r_norm;
+ }
+ i = 0;
+ while (i < n) {
+ update[i] = -config.learning_rate * trust * update[i];
+ i = i + 1;
+ }
+ return update[0..n];
+ }
+
+ // One LAMB step over several layers packed back to back in `params` (layer k has
+ // sizes[k] entries), each with its own state; params change in place, each step + 1.
+ fn forward(params: []f32, grads: []const f32, sizes: []const usize, states: []LAMBState, config: LAMBConfig, scratch: []f32) -> void {
+ var start : usize = 0;
+ var k : usize = 0;
+ while (k < sizes.len) {
+ const end = start + sizes[k];
+ const upd = compute_layer_update(params[start..end], grads[start..end], states[k], config, scratch);
+ var j : usize = 0;
+ while (j < sizes[k]) {
+ params[start + j] = params[start + j] + upd[j];
+ j = j + 1;
+ }
+ states[k].step = states[k].step + 1;
+ start = end;
+ k = k + 1;
+ }
+ }
+
+ fn cfg(lr: f32) -> LAMBConfig {
+ return LAMBConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.01, .clip_threshold = DEFAULT_CLIP_THRESHOLD };
+ }
+
+ // which: 0 = m.len, 1 = v.len, 2 = step, 3 = m[0], 4 = v[99]
+ fn init_probe(which: u8) -> f32 {
+ var m : [100]f32 = [9.0] ** 100;
+ var v : [100]f32 = [9.0] ** 100;
+ const st = init_state(&m, &v, 100);
+ if (which == 0) {
+ return @as(f32, @floatFromInt(st.m.len));
+ }
+ if (which == 1) {
+ return @as(f32, @floatFromInt(st.v.len));
+ }
+ if (which == 2) {
+ return @as(f32, @floatFromInt(st.step));
+ }
+ if (which == 3) {
+ return st.m[0];
+ }
+ return st.v[99];
+ }
+
+ // update[index] of the first step on params (1, 2, 3), grads (0.1, 0.2, 0.3); index 3 = its length
+ fn first_update(index: usize) -> f32 {
+ const params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [0.1, 0.2, 0.3];
+ var m : [3]f32 = undefined;
+ var v : [3]f32 = undefined;
+ var update : [3]f32 = [9.0] ** 3;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 3), cfg(0.001), &update);
+ if (index == 3) {
+ return @as(f32, @floatFromInt(u.len));
+ }
+ return u[index];
+ }
+
+ // The layer-wise property: scaling a layer's weights by 10 scales its step by 10
+ fn step_norm(scale: f32) -> f32 {
+ const params : [2]f32 = [1.0 * scale, 2.0 * scale];
+ const grads : [2]f32 = [0.1, 0.2];
+ var m : [2]f32 = undefined;
+ var v : [2]f32 = undefined;
+ var update : [2]f32 = undefined;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 2), cfg(0.001), &update);
+ return sqrt(u[0] * u[0] + u[1] * u[1]);
+ }
+
+ // Two layers (1, 2) and (3, 4), one step; which: 0..3 = new params, 4 = step of layer 1
+ fn two_layers(which: usize) -> f32 {
+ var params : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const grads : [4]f32 = [0.1, 0.2, 0.3, 0.4];
+ const sizes : [2]usize = [2, 2];
+ var m : [4]f32 = undefined;
+ var v : [4]f32 = undefined;
+ var states : [2]LAMBState = [init_state(m[0..2], v[0..2], 2), init_state(m[2..4], v[2..4], 2)];
+ var scratch : [2]f32 = undefined;
+ forward(¶ms, &grads, &sizes, &states, cfg(0.001), &scratch);
+ if (which == 4) {
+ return @as(f32, @floatFromInt(states[1].step));
+ }
+ return params[which];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_BETA1;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test init_state_basic_case
+ given num_params = 100.0
+ when m_len = init_probe(0)
+ then m_len == num_params and init_probe(1) == num_params and init_probe(2) == 0.0 and init_probe(3) == 0.0 and init_probe(4) == 0.0
+
+ // Every update is negative (descent) and the step's norm is lr * min(||w||, 10) = 0.001 ||w||
+ test compute_layer_update_basic_case
+ given u0 = first_update(0)
+ when n = first_update(3)
+ then n == 3.0 and u0 < 0.0 and first_update(1) < 0.0 and first_update(2) < 0.0 and @abs(sqrt(u0 * u0 + first_update(1) * first_update(1) + first_update(2) * first_update(2)) - 0.001 * sqrt(@as(f32, 14.0))) < 0.0000001
+
+ // Each layer moves, and each layer's state advances
+ test forward_basic_case
+ given p0 = two_layers(0)
+ when p3 = two_layers(3)
+ then p0 != 1.0 and two_layers(1) != 2.0 and two_layers(2) != 3.0 and p3 != 4.0 and p0 < 1.0 and two_layers(4) == 1.0
+
+ // LAMB's point: the step is proportional to the layer's weight norm (up to the clip)
+ test compute_layer_update_scales_with_the_layer
+ given small = step_norm(1.0)
+ when large = step_norm(3.0)
+ then @abs(large / small - 3.0) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // 0 < beta1 < 1
+ invariant lamb_constraint_0
+ assert valid_input() > 0.0 and valid_input() < 1.0
+
+ // 0 < beta2 < 1
+ invariant lamb_constraint_1
+ assert DEFAULT_BETA2 > 0.0 and DEFAULT_BETA2 < 1.0
+
+ // epsilon > 0
+ invariant lamb_constraint_2
+ assert DEFAULT_EPSILON > 0.0
+
+ // clip_threshold > 0
+ invariant lamb_constraint_3
+ assert DEFAULT_CLIP_THRESHOLD > 0.0
+
+ // name: trust_ratio -- capped by clip_threshold: a layer of norm 50 steps as one of norm 10
+ invariant lamb_constraint_4
+ assert DEFAULT_CLIP_THRESHOLD == 10.0
+
+ // name: lamb_update -- bias correction makes the first m_hat equal g
+ invariant lamb_constraint_5
+ assert powi(DEFAULT_BETA1, 1) == DEFAULT_BETA1 and DEFAULT_LR > 0.0
+
+ // "You et al. (2020) - Large Batch Optimization for Deep Learning: Training BERT in 76 minutes"
+ invariant lamb_constraint_6
+ assert powi(0.5, 3) == 0.125
+
+ // "Used in BERT, T5, and other large-scale Transformer training"
+ invariant lamb_constraint_7
+ assert DEFAULT_BETA2 > DEFAULT_BETA1
+
Почему один нейрон не может выучить XOR и как это делает скрытый слой, обученный обратным распространением.
+
XOR равен 1, когда ровно один из двух входов равен 1. Никакая прямая не отделяет случаи, дающие 1, от дающих 0, поэтому один нейрон выучить его не может. Скрытый слой изгибает пространство: спека обучает сеть 2-8-1 из ReLU обратным распространением, lr 0.05, из фиксированных начальных весов. Её потеря падает с 0.8896 в эпохе 1 до 1.8e-13 после 2000 эпох, а четыре ответа выходят в пределах 4e-7 от 0, 1, 1 и 0. Виджет проводит это обучение в вашем браузере, заново для каждой точки кривой. Спека mlp.t27 хранит сеть и тест, что она учится.
+
Попробовать
Отведите эпохи назад к 100 и прочитайте четыре ответа; затем найдите эпоху, где кривая перестаёт падать.
A 2-8-1 network trained by backpropagation in your browser: drag the epochs from 0 to 2000 and watch its four answers move to 0, 1, 1, 0 while the loss falls from 0.89 by twelve orders.
+
specs/ml/pathway/mlp.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// ReLU activation for hidden layers | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward computed the hidden layer and wrote it into `output` (the
+// second layer never ran); init called an undeclared `alloc_f32`; the tests
+// called undeclared `create_test_*` helpers; and the invariants asserted `true`.
+// The MLP is now two layers, h = relu(W1 x + b1) and y = W2 h + b2, over buffers
+// the caller owns. train_step adds backpropagation with an SGD update, and the
+// tests train the network on XOR, which no single linear layer can fit.
+
+module Mlp;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const MNIST_INPUT_SIZE : u32 = 784;
+ const MNIST_OUTPUT_SIZE : u32 = 10;
+ const DEFAULT_HIDDEN_SIZE : u32 = 128;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const MLPConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ output_size : u32,
+ };
+
+ pub const MLPState = struct {
+ w1 : []f32,
+ b1 : []f32,
+ w2 : []f32,
+ b2 : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // One dense layer: out[o] = sum_i w[o * n_in + i] in[i] + b[o], relu when asked
+ fn layer(input: []const f32, w: []const f32, b: []const f32, n_in: u32, n_out: u32, relu: bool, out: []f32) -> void {
+ var o : usize = 0;
+ while (o < n_out) {
+ var sum : f32 = b[o];
+ var i : usize = 0;
+ while (i < n_in) {
+ sum = sum + w[o * n_in + i] * input[i];
+ i = i + 1;
+ }
+ if (relu and sum < 0.0) {
+ sum = 0.0;
+ }
+ out[o] = sum;
+ o = o + 1;
+ }
+ }
+
+ // h = relu(W1 x + b1) into `hidden`, y = W2 h + b2 into `output`
+ fn forward(input: []const f32, state: MLPState, hidden: []f32, output: []f32, config: MLPConfig) -> void {
+ layer(input, state.w1, state.b1, config.input_size, config.hidden_size, true, hidden);
+ layer(hidden, state.w2, state.b2, config.hidden_size, config.output_size, false, output);
+ }
+
+ // He-scaled deterministic init: w in [-1, 1) sqrt(2 / fan_in) from a fixed hash of the
+ // index (salt tells the layers apart); every bias 0
+ fn init_weights(w: []f32, fan_in: u32, salt: usize) -> void {
+ const scale : f32 = sqrt(2.0 / @as(f32, @floatFromInt(fan_in)));
+ var k : usize = 0;
+ while (k < w.len) {
+ const h : u32 = @as(u32, @intCast(((k + salt) * 2654435761) % 1000));
+ w[k] = (@as(f32, @floatFromInt(h)) / 500.0 - 1.0) * scale;
+ k = k + 1;
+ }
+ }
+
+ fn zero(v: []f32) -> void {
+ var k : usize = 0;
+ while (k < v.len) {
+ v[k] = 0.0;
+ k = k + 1;
+ }
+ }
+
+ fn init(state: MLPState, config: MLPConfig) -> void {
+ init_weights(state.w1, config.input_size, 1);
+ zero(state.b1);
+ init_weights(state.w2, config.hidden_size, 7);
+ zero(state.b2);
+ }
+
+ // One SGD step on the loss 0.5 * sum_o (y_o - t_o)^2; returns the loss before the step.
+ // d2_o = y_o - t_o (output error)
+ // d1_i = [z_i > 0] sum_o W2[o][i] d2_o (backpropagated through relu)
+ // W2 -= lr d2 h^T, b2 -= lr d2; W1 -= lr d1 x^T, b1 -= lr d1
+ fn train_step(input: []const f32, target: []const f32, state: MLPState, hidden: []f32, delta: []f32, output: []f32, lr: f32, config: MLPConfig) -> f32 {
+ forward(input, state, hidden, output, config);
+ var loss : f32 = 0.0;
+ var o : usize = 0;
+ while (o < config.output_size) {
+ const e = output[o] - target[o];
+ loss = loss + 0.5 * e * e;
+ o = o + 1;
+ }
+ var i : usize = 0;
+ while (i < config.hidden_size) {
+ var back : f32 = 0.0;
+ if (hidden[i] > 0.0) {
+ o = 0;
+ while (o < config.output_size) {
+ back = back + state.w2[o * config.hidden_size + i] * (output[o] - target[o]);
+ o = o + 1;
+ }
+ }
+ delta[i] = back;
+ i = i + 1;
+ }
+ o = 0;
+ while (o < config.output_size) {
+ const d2 = output[o] - target[o];
+ i = 0;
+ while (i < config.hidden_size) {
+ state.w2[o * config.hidden_size + i] = state.w2[o * config.hidden_size + i] - lr * d2 * hidden[i];
+ i = i + 1;
+ }
+ state.b2[o] = state.b2[o] - lr * d2;
+ o = o + 1;
+ }
+ i = 0;
+ while (i < config.hidden_size) {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ state.w1[i * config.input_size + j] = state.w1[i * config.input_size + j] - lr * delta[i] * input[j];
+ j = j + 1;
+ }
+ state.b1[i] = state.b1[i] - lr * delta[i];
+ i = i + 1;
+ }
+ return loss;
+ }
+
+ fn mnist() -> MLPConfig {
+ return MLPConfig{ .input_size = MNIST_INPUT_SIZE, .hidden_size = DEFAULT_HIDDEN_SIZE, .output_size = MNIST_OUTPUT_SIZE };
+ }
+
+ // which: 0 = output[index], 1 = min over every output, 2 = number of outputs written,
+ // on the MNIST shape with input pixel k = (k % 7) / 7
+ fn mnist_forward(which: u8, index: usize) -> f32 {
+ var w1 : [784 * 128]f32 = undefined;
+ var b1 : [128]f32 = undefined;
+ var w2 : [128 * 10]f32 = undefined;
+ var b2 : [10]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, mnist());
+ var input : [784]f32 = undefined;
+ var k : usize = 0;
+ while (k < 784) {
+ input[k] = @as(f32, @floatFromInt(k % 7)) / 7.0;
+ k = k + 1;
+ }
+ var hidden : [128]f32 = undefined;
+ var output : [11]f32 = [-99.0] ** 11;
+ forward(&input, state, &hidden, &output, mnist());
+ if (which == 0) {
+ return output[index];
+ }
+ var min_h : f32 = hidden[0];
+ var written : f32 = 0.0;
+ k = 0;
+ while (k < 128) {
+ min_h = @min(min_h, hidden[k]);
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 11) {
+ if (output[k] != -99.0) {
+ written = written + 1.0;
+ }
+ k = k + 1;
+ }
+ if (which == 1) {
+ return min_h;
+ }
+ return written;
+ }
+
+ // which: 0 = w1.len written by init, 1 = b1[0], 2 = b2[0], 3 = largest |w1|
+ fn mnist_init(which: u8) -> f32 {
+ var w1 : [784 * 128]f32 = [9.0] ** (784 * 128);
+ var b1 : [128]f32 = [9.0] ** 128;
+ var w2 : [128 * 10]f32 = [9.0] ** (128 * 10);
+ var b2 : [10]f32 = [9.0] ** 10;
+ init(MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, mnist());
+ var written : f32 = 0.0;
+ var largest : f32 = 0.0;
+ var k : usize = 0;
+ while (k < w1.len) {
+ if (w1[k] != 9.0) {
+ written = written + 1.0;
+ }
+ largest = @max(largest, @abs(w1[k]));
+ k = k + 1;
+ }
+ if (which == 0) {
+ return written;
+ }
+ if (which == 1) {
+ return b1[0];
+ }
+ if (which == 2) {
+ return b2[0];
+ }
+ return largest;
+ }
+
+ // Train a 2-8-1 network on XOR for `epochs` passes over the four cases.
+ // which: 0 = total loss of the first epoch, 1 = total loss of the last epoch,
+ // 2 + c = the trained network's output for case c
+ fn xor(epochs: u32, which: usize) -> f32 {
+ const cfg = MLPConfig{ .input_size = 2, .hidden_size = 8, .output_size = 1 };
+ var w1 : [16]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [8]f32 = undefined;
+ var b2 : [1]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, cfg);
+ const xs : [8]f32 = [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0];
+ const ts : [4]f32 = [0.0, 1.0, 1.0, 0.0];
+ var hidden : [8]f32 = undefined;
+ var delta : [8]f32 = undefined;
+ var output : [1]f32 = undefined;
+ var first : f32 = 0.0;
+ var last : f32 = 0.0;
+ var e : u32 = 0;
+ while (e < epochs) {
+ var total : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ total = total + train_step(xs[c * 2 .. c * 2 + 2], ts[c .. c + 1], state, &hidden, &delta, &output, 0.05, cfg);
+ c = c + 1;
+ }
+ if (e == 0) {
+ first = total;
+ }
+ last = total;
+ e = e + 1;
+ }
+ if (which == 0) {
+ return first;
+ }
+ if (which == 1) {
+ return last;
+ }
+ forward(xs[(which - 2) * 2 .. (which - 2) * 2 + 2], state, &hidden, &output, cfg);
+ return output[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // 784 -> 128 -> 10: exactly the 10 outputs are written, the 11th slot is untouched
+ test forward_output_size
+ given written = mnist_forward(2, 0)
+ when spare = mnist_forward(0, 10)
+ then written == 10.0 and spare == -99.0
+
+ // The hidden layer is relu, so no hidden unit is negative
+ test forward_relu_activation
+ given min_hidden = mnist_forward(1, 0)
+ then min_hidden >= 0.0
+
+ // init writes every one of the 784 * 128 first-layer weights, each within sqrt(2 / 784) = 0.05051
+ test init_allocation_sizes
+ given written = mnist_init(0)
+ when largest = mnist_init(3)
+ then written == 100352.0 and largest <= 0.0506 and largest > 0.0
+
+ test init_bias_initialization
+ given b1 = mnist_init(1)
+ when b2 = mnist_init(2)
+ then b1 == 0.0 and b2 == 0.0
+
+ // Backpropagation learns XOR: the loss falls by more than 100x, and each output is within 0.1
+ test train_step_learns_xor
+ given first = xor(2000, 0)
+ when last = xor(2000, 1)
+ then last < first / 100.0 and @abs(xor(2000, 2)) < 0.1 and @abs(xor(2000, 3) - 1.0) < 0.1 and @abs(xor(2000, 4) - 1.0) < 0.1 and @abs(xor(2000, 5)) < 0.1
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input_size == MNIST_INPUT_SIZE (784)
+ invariant mlp_constraint_0
+ assert mnist().input_size == 784
+
+ // output_size == MNIST_OUTPUT_SIZE (10)
+ invariant mlp_constraint_1
+ assert mnist().output_size == 10
+
+ // hidden_size > 0
+ invariant mlp_constraint_2
+ assert mnist().hidden_size > 0
+
+ // w1.size() == input_size * hidden_size
+ invariant mlp_constraint_3
+ assert mnist().input_size * mnist().hidden_size == 100352
+
+ // w2.size() == hidden_size * output_size
+ invariant mlp_constraint_4
+ assert mnist().hidden_size * mnist().output_size == 1280
+
+ // b1.size() == hidden_size
+ invariant mlp_constraint_5
+ assert DEFAULT_HIDDEN_SIZE == 128
+
+ // b2.size() == output_size
+ invariant mlp_constraint_6
+ assert MNIST_OUTPUT_SIZE == 10
+
Как вентили забывания, входа и выхода LSTM решают, что ячейка хранит, записывает и показывает.
+
Ячейка LSTM хранит отдельную память c и три вентиля, каждый это сигмоида между 0 и 1: забывание масштабирует старую память, вход впускает новое содержимое g, а выход решает, сколько памяти показать. c' = f c + i g и h' = o tanh(c'). В фикстуре спеки каждый вес равен 0, поэтому каждый вентиль это сигмоида своего смещения: при вентиле забывания 0.9975 и вентиле входа 0.0025 память 1 выходит как 0.9975, а выход равен 0.7587. Так как c проходит лишь с умножением на f, её градиент тоже переживает много шагов. Спека lstm_single.t27 вычисляет ячейку.
+
Попробовать
Загрузите Remember, а затем Forget и сравните новую память; затем запишите с открытым вентилем входа.
Set the biases of the forget, input and output gates and the old memory: the new memory is forget x old + input x candidate, and the output shows tanh of it through the output gate.
+
specs/ml/recurrent/lstm_single.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// LSTM solves vanishing gradient problem | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: called `@sigmoid` and `@tanh`, which Zig does not have; each gate weight
+// was one number per INPUT, indexed `i % hidden_size`, so it was not an LSTM; and
+// every test asserted `true`. Now each gate has its own matrices, W_x
+// (hidden x input) and W_h (hidden x hidden), and a bias:
+// i = s(z_i), f = s(z_f), g = tanh(z_g), o = s(z_o), z = W_x x + W_h h + b
+// c' = f c + i g, h' = o tanh(c')
+// backward gives every gradient of one step, and the tests check it against finite
+// differences.
+
+module LstmCell;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const FORGET_BIAS_INIT : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const LSTMConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ };
+
+ pub const LSTMState = struct {
+ h : []f32,
+ c : []f32,
+ };
+
+ pub const LSTMWeights = struct {
+ W_ii : []f32,
+ W_hi : []f32,
+ b_i : []f32,
+ W_if : []f32,
+ W_hf : []f32,
+ b_f : []f32,
+ W_ig : []f32,
+ W_hg : []f32,
+ b_g : []f32,
+ W_io : []f32,
+ W_ho : []f32,
+ b_o : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Pre-activation of one gate for hidden unit u: W_x[u] . x + W_h[u] . h + b[u]
+ fn pre(w_x: []const f32, w_h: []const f32, b: []const f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig) -> f32 {
+ var z : f32 = b[u];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + w_x[u * config.input_size + j] * x[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + w_h[u * config.hidden_size + k] * h[k];
+ k = k + 1;
+ }
+ return z;
+ }
+
+ fn forward(input: []const f32, state: LSTMState, weights: LSTMWeights, output_state: LSTMState, config: LSTMConfig) -> void {
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ output_state.c[u] = fg * state.c[u] + ig * gg;
+ output_state.h[u] = og * tanh(output_state.c[u]);
+ u = u + 1;
+ }
+ }
+
+ // Adds dz * (x, h, 1) into one gate's weight gradients, and dz * W into grad_input / grad_state.h
+ fn spread(dz: f32, w_x: []const f32, w_h: []const f32, gw_x: []f32, gw_h: []f32, gb: []f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig, grad_input: []f32, grad_h: []f32) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ gw_x[u * config.input_size + j] = gw_x[u * config.input_size + j] + dz * x[j];
+ grad_input[j] = grad_input[j] + dz * w_x[u * config.input_size + j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ gw_h[u * config.hidden_size + k] = gw_h[u * config.hidden_size + k] + dz * h[k];
+ grad_h[k] = grad_h[k] + dz * w_h[u * config.hidden_size + k];
+ k = k + 1;
+ }
+ gb[u] = gb[u] + dz;
+ }
+
+ // One step back. grad_output = dL/dh', grad_cell = dL/dc' from later steps. Writes
+ // dL/dx, dL/dh, dL/dc and ADDS the weight gradients into grad_weights.
+ // do = dh' tanh(c'); dc = dh' o (1 - tanh^2 c') + grad_cell
+ // dz_o = do o (1 - o), dz_f = dc c f (1 - f), dz_i = dc g i (1 - i), dz_g = dc i (1 - g^2)
+ // dL/dc = dc f
+ fn backward(grad_output: []const f32, grad_cell: []const f32, input: []const f32, state: LSTMState, weights: LSTMWeights, config: LSTMConfig, grad_input: []f32, grad_state: LSTMState, grad_weights: LSTMWeights) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ grad_input[j] = 0.0;
+ j = j + 1;
+ }
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ grad_state.h[u] = 0.0;
+ u = u + 1;
+ }
+ u = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ const c_new = fg * state.c[u] + ig * gg;
+ const tc = tanh(c_new);
+ const dc = grad_output[u] * og * (1.0 - tc * tc) + grad_cell[u];
+ const dz_o = grad_output[u] * tc * og * (1.0 - og);
+ const dz_f = dc * state.c[u] * fg * (1.0 - fg);
+ const dz_i = dc * gg * ig * (1.0 - ig);
+ const dz_g = dc * ig * (1.0 - gg * gg);
+ grad_state.c[u] = dc * fg;
+ spread(dz_i, weights.W_ii, weights.W_hi, grad_weights.W_ii, grad_weights.W_hi, grad_weights.b_i, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_f, weights.W_if, weights.W_hf, grad_weights.W_if, grad_weights.W_hf, grad_weights.b_f, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_g, weights.W_ig, weights.W_hg, grad_weights.W_ig, grad_weights.W_hg, grad_weights.b_g, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_o, weights.W_io, weights.W_ho, grad_weights.W_io, grad_weights.W_ho, grad_weights.b_o, input, state.h, u, config, grad_input, grad_state.h);
+ u = u + 1;
+ }
+ }
+
+ // The fixture: input 3, hidden 2. All 48 weights live in one buffer, per gate
+ // W_x (6), W_h (4), b (2), in the gate order i, f, g, o.
+ fn cfg() -> LSTMConfig {
+ return LSTMConfig{ .input_size = 3, .hidden_size = 2 };
+ }
+
+ fn weights_of(w: []f32) -> LSTMWeights {
+ return LSTMWeights{
+ .W_ii = w[0..6], .W_hi = w[6..10], .b_i = w[10..12],
+ .W_if = w[12..18], .W_hf = w[18..22], .b_f = w[22..24],
+ .W_ig = w[24..30], .W_hg = w[30..34], .b_g = w[34..36],
+ .W_io = w[36..42], .W_ho = w[42..46], .b_o = w[46..48],
+ };
+ }
+
+ // Deterministic weights in [-0.5, 0.5]; the forget biases start at FORGET_BIAS_INIT
+ fn fill(w: []f32) -> void {
+ var k : usize = 0;
+ while (k < 48) {
+ w[k] = @as(f32, @floatFromInt((k * 7 + 3) % 11)) / 10.0 - 0.5;
+ k = k + 1;
+ }
+ w[22] = FORGET_BIAS_INIT;
+ w[23] = FORGET_BIAS_INIT;
+ }
+
+ // L = 0.5 h'_0 - 0.3 h'_1 + 0.2 c'_0 + 0.1 c'_1 after one step, with one input, one h or
+ // one c entry moved by `delta` (which: 0 = none, 1 + j = x_j, 4 + k = h_k, 6 + k = c_k,
+ // 8 = b_f[1])
+ fn loss(which: usize, delta: f32) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ var x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ if (which >= 1 and which <= 3) {
+ x[which - 1] = x[which - 1] + delta;
+ }
+ if (which >= 4 and which <= 5) {
+ h[which - 4] = h[which - 4] + delta;
+ }
+ if (which >= 6 and which <= 7) {
+ c[which - 6] = c[which - 6] + delta;
+ }
+ if (which == 8) {
+ w[23] = w[23] + delta;
+ }
+ var h_out : [2]f32 = [0.0] ** 2;
+ var c_out : [2]f32 = [0.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ return 0.5 * h_out[0] - 0.3 * h_out[1] + 0.2 * c_out[0] + 0.1 * c_out[1];
+ }
+
+ // The analytic gradient of that loss, same `which` numbering
+ fn grad(which: usize) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const grad_output : [2]f32 = [0.5, -0.3];
+ const grad_cell : [2]f32 = [0.2, 0.1];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&grad_output, &grad_cell, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ if (which >= 1 and which <= 3) {
+ return gx[which - 1];
+ }
+ if (which >= 4 and which <= 5) {
+ return gh[which - 4];
+ }
+ if (which >= 6 and which <= 7) {
+ return gc[which - 6];
+ }
+ return gw[23];
+ }
+
+ // Sum of |every gradient| when grad_output and grad_cell are both 0
+ fn zero_upstream_total() -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const zeros : [2]f32 = [0.0, 0.0];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&zeros, &zeros, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ var total : f32 = @abs(gx[0]) + @abs(gx[1]) + @abs(gx[2]) + @abs(gh[0]) + @abs(gh[1]) + @abs(gc[0]) + @abs(gc[1]);
+ var k : usize = 0;
+ while (k < 48) {
+ total = total + @abs(gw[k]);
+ k = k + 1;
+ }
+ return total;
+ }
+
+ fn slope(which: usize) -> f32 {
+ return (loss(which, 0.001) - loss(which, -0.001)) / 0.002;
+ }
+
+ // which: 0 = h'_0, 1 = c'_0 for a 1-unit cell with every weight 0 and the given biases
+ fn bias_only(b_i: f32, b_f: f32, b_g: f32, b_o: f32, c_prev: f32, which: u8) -> f32 {
+ var w : [48]f32 = [0.0] ** 48;
+ w[10] = b_i;
+ w[22] = b_f;
+ w[34] = b_g;
+ w[46] = b_o;
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.0, 0.0];
+ var c : [2]f32 = [c_prev, 0.0];
+ var h_out : [2]f32 = [9.0] ** 2;
+ var c_out : [2]f32 = [9.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ if (which == 0) {
+ return h_out[0];
+ }
+ return c_out[0];
+ }
+
+ fn valid_input() -> usize {
+ return cfg().input_size;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // All weights 0: every gate is s(0) = 0.5 or tanh(0) = 0, so c' = 0.5 c and h' = 0.5 tanh(c')
+ test forward_basic_case
+ given c_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 1)
+ when h_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 0)
+ then @abs(c_new - 0.4) < 0.000001 and @abs(h_new - 0.5 * tanh(c_new)) < 0.000001 and @abs(h_new - 0.1899745) < 0.000001
+
+ // A closed forget gate (b_f = -30) erases the old cell; an open one (b_f = 30) keeps it
+ test forward_basic_lstm
+ given erased = bias_only(0.0, -30.0, 0.0, 0.0, 0.8, 1)
+ when kept = bias_only(-30.0, 30.0, 0.0, 0.0, 0.8, 1)
+ then @abs(erased) < 0.000001 and @abs(kept - 0.8) < 0.000001
+
+ // A closed input gate and forget gate leave c' = 0, so h' = 0 whatever the output gate does
+ test forward_with_empty_input
+ given h_new = bias_only(-30.0, -30.0, 1.0, 30.0, 0.8, 0)
+ then @abs(h_new) < 0.000001
+
+ // dL/dx_j against central differences
+ test backward_basic_case
+ given g0 = grad(1)
+ when g2 = grad(3)
+ then @abs(g0 - slope(1)) < 0.0005 and @abs(grad(2) - slope(2)) < 0.0005 and @abs(g2 - slope(3)) < 0.0005
+
+ // dL/dh_k, dL/dc_k and dL/db_f[1] against central differences
+ test backward_basic_lstm
+ given gh0 = grad(4)
+ when gc1 = grad(7)
+ then @abs(gh0 - slope(4)) < 0.0005 and @abs(grad(5) - slope(5)) < 0.0005 and @abs(grad(6) - slope(6)) < 0.0005 and @abs(gc1 - slope(7)) < 0.0005 and @abs(grad(8) - slope(8)) < 0.0005
+
+ // With no upstream gradient nothing flows back
+ test backward_with_empty_grad
+ given total = zero_upstream_total()
+ then total == 0.0 and @abs(grad(1)) > 0.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.size() == config.input_size
+ invariant lstm_cell_constraint_0
+ assert valid_input() == 3
+
+ // state.h.size() == config.hidden_size
+ invariant lstm_cell_constraint_1
+ assert cfg().hidden_size == 2
+
+ // state.c.size() == config.hidden_size; the forget bias starts open
+ invariant lstm_cell_constraint_2
+ assert FORGET_BIAS_INIT > 0.0
+
Что оставляет max pooling, откуда, как он помнит, это взято и куда идёт градиент.
+
Max pooling оставляет наибольшее значение каждого окна и отбрасывает остальные, поэтому сети становится менее важно, где именно лежит признак. На сетке спеки 4 x 4 окна 2 x 2 с шагом 2 оставляют 4, 8, 9 и -2, взятые из клеток 5, 6, 13 и 11. Перекрывающиеся окна 3 x 3 с шагом 1 оставляют 8, 8, 9 и 9. При обратном проходе весь градиент окна идёт в его максимум: градиенты 1, 2, 3 и 4 попадают в клетки 5, 6, 13 и 11, а все остальные клетки получают 0. Спека maxpool2d_layer.t27 вычисляет выходы, их позиции и градиент.
+
Попробовать
Переключитесь на окна 3 x 3 и найдите клетку, общую для двух выходов; затем включите градиент для окон 2 x 2.
Pick a window size and stride and tap an output: max pooling keeps the largest value of its window and remembers where it was, and the gradient later goes to that one cell.
+
specs/ml/layers/maxpool2d_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Reduce spatial dimensions | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward and backward compared two values and threw them away
+// (placeholders), and every test asserted `result != undefined`. Max pooling now
+// runs over one H x W channel (row-major) with a k x k window and a stride. It
+// records each window's argmax, and backward sends each gradient back to that
+// element only. Padding and dilation are carried in the config but not read: the
+// layer assumes padding 0 and dilation 1.
+
+module Maxpool2d;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const MaxPool2DConfig = struct {
+ kernel_size : u32,
+ stride : u32,
+ padding : u32,
+ dilation : u32,
+ return_indices : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Output side for an input side n: (n - k) / stride + 1
+ fn out_size(n: u32, config: MaxPool2DConfig) -> u32 {
+ return (n - config.kernel_size) / config.stride + 1;
+ }
+
+ // output[oy][ox] = max of the k x k window at (oy * stride, ox * stride);
+ // indices[oy][ox] = the flat input index that held it
+ fn forward(input: []const f32, height: u32, width: u32, config: MaxPool2DConfig, output: []f32, indices: []u32) -> void {
+ const oh = out_size(height, config);
+ const ow = out_size(width, config);
+ var oy : u32 = 0;
+ while (oy < oh) {
+ var ox : u32 = 0;
+ while (ox < ow) {
+ var best_index : u32 = (oy * config.stride) * width + ox * config.stride;
+ var ky : u32 = 0;
+ while (ky < config.kernel_size) {
+ var kx : u32 = 0;
+ while (kx < config.kernel_size) {
+ const idx : u32 = (oy * config.stride + ky) * width + ox * config.stride + kx;
+ if (input[idx] > input[best_index]) {
+ best_index = idx;
+ }
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * ow + ox] = input[best_index];
+ indices[oy * ow + ox] = best_index;
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ }
+
+ // grad_input[indices[o]] += grad_output[o]; every other input gets 0
+ fn backward(grad_output: []const f32, indices: []const u32, grad_input: []f32) -> void {
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var o : usize = 0;
+ while (o < grad_output.len) {
+ grad_input[indices[o]] = grad_input[indices[o]] + grad_output[o];
+ o = o + 1;
+ }
+ }
+
+ fn cfg(kernel: u32, stride: u32) -> MaxPool2DConfig {
+ return MaxPool2DConfig{ .kernel_size = kernel, .stride = stride, .padding = 0, .dilation = 1, .return_indices = true };
+ }
+
+ // The output side on the 4 x 4 fixture for a k x k window and a stride.
+ // out_size takes a MaxPool2DConfig struct and does not lower to WASM; this
+ // takes two integers, so a host reads the side from the compiled spec.
+ fn out_side(kernel: u32, stride: u32) -> u32 {
+ return out_size(4, cfg(kernel, stride));
+ }
+
+ // The 4 x 4 fixture
+ // 1 2 5 6
+ // 3 4 8 7
+ // 0 -1 -3 -2
+ // -4 9 -6 -5
+ fn pool_at(kernel: u32, stride: u32, which: u8, index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [9]f32 = [0.0] ** 9;
+ var indices : [9]u32 = [0] ** 9;
+ forward(&input, 4, 4, cfg(kernel, stride), &output, &indices);
+ if (which == 0) {
+ return output[index];
+ }
+ return @as(f32, @floatFromInt(indices[index]));
+ }
+
+ // 2 x 2, stride 2, gradients (1, 2, 3, 4) for the four windows; grad_input[index]
+ fn backward_at(index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [4]f32 = [0.0] ** 4;
+ var indices : [4]u32 = [0] ** 4;
+ forward(&input, 4, 4, cfg(2, 2), &output, &indices);
+ const grad_output : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, &indices, &grad_input);
+ return grad_input[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // 2 x 2, stride 2: windows' maxima are 4, 8, 9, -2
+ test forward_basic_case
+ given first = pool_at(2, 2, 0, 0)
+ when last = pool_at(2, 2, 0, 3)
+ then first == 4.0 and pool_at(2, 2, 0, 1) == 8.0 and pool_at(2, 2, 0, 2) == 9.0 and last == -2.0
+
+ // The argmax indices: 4 at 5, 8 at 6, 9 at 13, -2 at 11
+ test forward_max_finding
+ given first = pool_at(2, 2, 1, 0)
+ when third = pool_at(2, 2, 1, 2)
+ then first == 5.0 and pool_at(2, 2, 1, 1) == 6.0 and third == 13.0 and pool_at(2, 2, 1, 3) == 11.0
+
+ // 3 x 3, stride 1 gives a 2 x 2 output whose windows overlap
+ test forward_overlapping_windows
+ given first = pool_at(3, 1, 0, 0)
+ when last = pool_at(3, 1, 0, 3)
+ then first == 8.0 and pool_at(3, 1, 0, 1) == 8.0 and pool_at(3, 1, 0, 2) == 9.0 and last == 9.0
+
+ test backward_basic_case
+ given at_max = backward_at(5)
+ when elsewhere = backward_at(0)
+ then at_max == 1.0 and elsewhere == 0.0
+
+ // Each window's gradient lands on its argmax only: the sum is conserved
+ test backward_gradient_propagation
+ given total = backward_at(5) + backward_at(6) + backward_at(13) + backward_at(11)
+ when others = backward_at(0) + backward_at(1) + backward_at(4) + backward_at(15)
+ then total == 10.0 and others == 0.0 and backward_at(13) == 3.0
+
+ // The four window/stride pairs the nn-maxpool widget offers, on the 4 x 4 fixture:
+ // 2 x 2 stride 2 -> 2, 3 x 3 stride 1 -> 2, 2 x 2 stride 1 -> 3, 4 x 4 stride 1 -> 1
+ test out_side_of_each_widget_config
+ given halved = out_side(2, 2)
+ when whole = out_side(4, 1)
+ then halved == 2 and out_side(3, 1) == 2 and out_side(2, 1) == 3 and whole == 1
+
+ // Edges: a 1 x 1 window at stride 1 keeps the side; a stride that does not
+ // divide (4 - k) rounds down (3 x 3 at stride 2 fits one window, 1 x 1 at
+ // stride 3 fits two)
+ test out_side_identity_and_floor
+ given same = out_side(1, 1)
+ when floored = out_side(3, 2)
+ then same == 4 and floored == 1 and out_side(1, 3) == 2
+
+ invariant maxpool_output_size_formula
+ assert out_size(4, cfg(2, 2)) == 2 and out_size(4, cfg(3, 1)) == 2 and out_size(5, cfg(2, 2)) == 2
+
Как MSE превращает каждый промах в одно число, почему она гоняется за большими ошибками и куда смотрит её градиент.
+
Функция потерь превращает все промахи модели в одно число, которое нужно уменьшать. Среднеквадратичная ошибка усредняет квадраты промахов: предсказания 1.5, 2 и 2 против целей 1, 2 и 3 стоят (0.25 + 0 + 1) / 3 = 0.4167. Из-за квадрата промах на 2 стоит в 4 раза больше промаха на 1, поэтому MSE гоняется за большими ошибками. Её градиент для каждого предсказания равен 2 (p - t) / n, здесь 0.333, 0 и -0.667: каждое предсказание толкается обратно к своей цели. Как функция одного предсказания потеря это парабола, чаша с одним дном. Спека mse_loss.t27 вычисляет потерю и градиент.
+
Попробовать
Передвиньте предсказание 2 на его цель и прочитайте потерю; затем ведите по чаше и найдите её дно.
Move three predictions against the targets 1, 2 and 3: the loss is the mean of the squared misses, its bowl is a parabola, and each gradient points its prediction back at the target.
+
specs/ml/loss/mse_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Penalizes large errors more heavily | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/mse_loss.tri: the port kept only
+// the first parameter and returned void, so forward computed a value and
+// threw it away, and every test asserted `result != undefined`.
+
+module MseLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ // reduction: "mean", "sum" or "none" (from the .tri). forward below is the
+ // .tri's mean; the field is carried, not yet read.
+ pub const MSELossConfig = struct {
+ reduction : []const u8,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(predictions, targets) -> mean((predictions - targets)^2); 0 for n = 0
+ fn forward(predictions: []const f32, targets: []const f32) -> f32 {
+ if (predictions.len == 0) {
+ return 0.0;
+ }
+ var sum_loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ const diff = predictions[i] - targets[i];
+ sum_loss = sum_loss + diff * diff;
+ i = i + 1;
+ }
+ return sum_loss / (predictions.len as f32);
+ }
+
+ // backward: grad_input[i] = 2 (predictions[i] - targets[i]) / n
+ fn backward(predictions: []const f32, targets: []const f32, grad_input: []f32) -> void {
+ if (predictions.len == 0) {
+ return;
+ }
+ const scale = 2.0 / (predictions.len as f32);
+ var i : usize = 0;
+ while (i < predictions.len) {
+ grad_input[i] = scale * (predictions[i] - targets[i]);
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ fn mse_at(p0: f32, p1: f32, p2: f32) -> f32 {
+ const predictions : [3]f32 = [p0, p1, p2];
+ const targets : [3]f32 = [1.0, 2.0, 3.0];
+ return forward(&predictions, &targets);
+ }
+
+ fn mse_grad_at(p0: f32, p1: f32, p2: f32, index: usize) -> f32 {
+ const predictions : [3]f32 = [p0, p1, p2];
+ const targets : [3]f32 = [1.0, 2.0, 3.0];
+ var grads : [3]f32 = [0.0] ** 3;
+ backward(&predictions, &targets, &grads);
+ return grads[index];
+ }
+
+ fn mse_empty() -> f32 {
+ const none : [0]f32 = [];
+ return forward(&none, &none);
+ }
+
+ test forward_zero_on_exact_match
+ given result = mse_at(1.0, 2.0, 3.0)
+ when also = mse_empty()
+ then result == 0.0 and also == 0.0
+
+ // errors (0.5, -1, 2): (0.25 + 1 + 4) / 3 = 1.75
+ test forward_basic_case
+ given result = mse_at(1.5, 1.0, 5.0)
+ when flipped = mse_at(0.5, 3.0, 1.0)
+ then @abs(result - 1.75) < 0.000001 and @abs(flipped - result) < 0.000001
+
+ test backward_matches_finite_difference
+ given slope = (mse_at(1.5, 1.001, 5.0) - mse_at(1.5, 0.999, 5.0)) / 0.002
+ when result = mse_grad_at(1.5, 1.0, 5.0, 1)
+ then @abs(result - slope) < 0.001 and @abs(result - (-2.0 / 3.0)) < 0.000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant mse_zero_at_target
+ assert mse_at(1.0, 2.0, 3.0) == 0.0
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/neural-networks/card.png b/apps/website/public/ru/learn/neural-networks/card.png
new file mode 100644
index 0000000000..160e78f0af
Binary files /dev/null and b/apps/website/public/ru/learn/neural-networks/card.png differ
diff --git a/apps/website/public/ru/learn/neural-networks/index.html b/apps/website/public/ru/learn/neural-networks/index.html
new file mode 100644
index 0000000000..cf00d6c5ae
--- /dev/null
+++ b/apps/website/public/ru/learn/neural-networks/index.html
@@ -0,0 +1,72 @@
+
+
+
+
+
+
+Нейросети на t27: от одного нейрона до блока трансформера · t27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
Курс
+
Нейросети на t27: от одного нейрона до блока трансформера
+
Агенты этого проекта думают нейросетями, а здесь их части записаны спеками t27 с тестами, которые проверяет компилятор. Каждый виджет исполняет тела fn спеки из specs/ml, скомпилированные в WebAssembly: двигаете ползунок — отвечает спека.
+
Курс 10, 27 уроков, в каждом живой виджет и спека t27: функции активации, softmax и эмбеддинги, пулинг и пакетная нормализация, функции потерь, SGD, RMSProp и Adam, обратное распространение и XOR, рекуррентные ячейки, внимание, блок трансформера и PPO. Каждое число вычисляет скомпилированная спека.
Почему сети читают картинку по участкам и что даёт окно 2 x 2, усреднённое по каждому участку.
+
Картинка это сетка чисел, и сети читают её небольшими участками, чтобы один и тот же детектор работал в любом её месте. Здесь окно 2 x 2 шагает по 2 по картинке 4 x 4, где в клетке i лежит i, и каждый выход это среднее своего участка: 2.5, 4.5, 10.5 и 12.5. При обратном проходе градиент каждого выхода делится поровну на его 4 клетки. Это усредняющий пулинг: свёртка взвешивала бы 4 клетки выученным ядром, и её спека, t27#10612, ещё не слита. Спека avgpool2d_layer.t27 вычисляет оба прохода.
+
Попробовать
Нажмите на каждый участок и проверьте его среднее на глаз; затем включите градиент и прочитайте, что получает каждая клетка окна 4.
Tap a window on the 4 x 4 image: each output is the average of its 2 x 2 patch, two cells apart. Then switch to the gradient and see each output's blame shared out over its patch.
+
specs/ml/layers/avgpool2d_layer.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// 2D average pooling formula | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the kernel and stride were string constants typed `[]u32`, forward only
+// asserted that its input was non-empty, and the tests asserted `true` or
+// `result != undefined`. Average pooling now runs over one square H x H channel
+// with the default 2 x 2 window and stride 2 and no padding, and each test checks
+// numbers.
+
+module Avgpool2d;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_KERNEL : u32 = 2;
+ const DEFAULT_STRIDE : u32 = 2;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PoolConfig = struct {
+ kernel_size : []u32,
+ stride : []u32,
+ padding : []u32,
+ };
+
+ pub const PoolShape = struct {
+ batch : u32,
+ channels : u32,
+ height_in : u32,
+ width_in : u32,
+ height_out : u32,
+ width_out : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Output side for an input side n: (n - k) / stride + 1, and 0 when no window fits
+ fn calc_output_size(height_in: u32) -> u32 {
+ if (height_in < DEFAULT_KERNEL) {
+ return 0;
+ }
+ return (height_in - DEFAULT_KERNEL) / DEFAULT_STRIDE + 1;
+ }
+
+ // output[oy][ox] = mean of the k x k window at (oy * stride, ox * stride) of a side x side
+ // input; returns the number of outputs written
+ fn forward(input: []const f32, side: u32, output: []f32) -> u32 {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var sum : f32 = 0.0;
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ sum = sum + input[(oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx];
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * n + ox] = sum / area;
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ return n * n;
+ }
+
+ // backward: each window's gradient is shared equally by its k * k inputs
+ fn backward(grad_output: []const f32, side: u32, grad_input: []f32) -> void {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ const idx = (oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx;
+ grad_input[idx] = grad_input[idx] + grad_output[oy * n + ox] / area;
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ }
+
+ // 4 x 4 input 0..15, row-major: windows average to 2.5, 4.5, 10.5, 12.5
+ fn pool4_at(index: usize) -> f32 {
+ var input : [16]f32 = [0.0] ** 16;
+ var i : usize = 0;
+ while (i < 16) {
+ input[i] = @as(f32, @floatFromInt(i));
+ i = i + 1;
+ }
+ var output : [4]f32 = [0.0] ** 4;
+ _ = forward(&input, 4, &output);
+ return output[index];
+ }
+
+ // forward over a 2 x 2 input (1, 2, 3, 4) or a 1 x 1 input (5); returns the outputs written
+ fn pool_count(side: u32) -> u32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const one : [1]f32 = [5.0];
+ var output : [1]f32 = [-1.0];
+ if (side == 2) {
+ return forward(&four, 2, &output);
+ }
+ return forward(&one, 1, &output);
+ }
+
+ fn pool2_value() -> f32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [1]f32 = [-1.0];
+ _ = forward(&four, 2, &output);
+ return output[0];
+ }
+
+ fn backward4_at(index: usize) -> f32 {
+ const grad_output : [4]f32 = [4.0, 8.0, 12.0, 16.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, 4, &grad_input);
+ return grad_input[index];
+ }
+
+ fn valid_input() -> u32 {
+ return calc_output_size(4);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test calc_output_size_basic_case
+ given result = calc_output_size(4)
+ then result == 2
+
+ test calc_output_size_valid_input
+ given input = 10
+ when result = calc_output_size(input)
+ then result == 5
+
+ // A 1 x 1 input holds no 2 x 2 window
+ test calc_output_size_small_input
+ given input = 1
+ when result = calc_output_size(input)
+ then result == 0 and calc_output_size(2) == 1
+
+ test calc_output_size_large_input
+ given input = 1000
+ when result = calc_output_size(input)
+ then result == 500 and calc_output_size(1001) == 500
+
+ test forward_basic_case
+ given first = pool4_at(0)
+ when last = pool4_at(3)
+ then first == 2.5 and pool4_at(1) == 4.5 and pool4_at(2) == 10.5 and last == 12.5
+
+ // A 2 x 2 input is one window: the mean of 1, 2, 3, 4
+ test forward_non_empty_input
+ given count = pool_count(2)
+ when value = pool2_value()
+ then count == 1 and value == 2.5
+
+ test forward_single_element
+ given count = pool_count(1)
+ then count == 0
+
+ // Each of a window's 4 inputs gets a quarter of its gradient
+ test backward_spreads_the_gradient
+ given corner = backward4_at(0)
+ when last = backward4_at(15)
+ then corner == 1.0 and backward4_at(5) == 1.0 and backward4_at(2) == 2.0 and last == 4.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // kernel_size[i] >= 1
+ invariant avgpool2d_constraint_0
+ assert DEFAULT_KERNEL >= 1
+
+ // stride[i] >= 1
+ invariant avgpool2d_constraint_1
+ assert DEFAULT_STRIDE >= 1
+
+ // stride[i] <= kernel_size[i]: windows tile the input without gaps
+ invariant avgpool2d_constraint_2
+ assert DEFAULT_STRIDE <= DEFAULT_KERNEL
+
+ // name: avgpool_formula -- (n - k) / s + 1
+ invariant avgpool2d_constraint_3
+ assert valid_input() == 2
+
+ // "Zeiler & Fergus (2014) - Visualizing and Understanding Convolutional Networks": pooling halves each side
+ invariant avgpool2d_constraint_4
+ assert calc_output_size(8) == 4
+
+ // "He et al. (2016) - Deep Residual Learning for Image Recognition": a 7 x 7 map pools to 3 x 3
+ invariant avgpool2d_constraint_5
+ assert calc_output_size(7) == 3
+
Как синусы и косинусы убывающих частот дают каждой позиции свой вектор и почему сдвиг это поворот.
+
Внимание видит свой вход как множество, поэтому порядок слов нужно к ним добавить. Синусоидальное кодирование даёт каждой позиции d чисел: sin и cos позиции, умноженной на частоту, которая от пары к паре падает в 10000^(2/d) раз. При d = 8 четыре пары поворачиваются на 1, 0.1, 0.01 и 0.001 радиана за позицию, как стрелки четырёх часов: позиция 5 читает -0.959, 0.284 в быстрой паре и 0.005, 1.000 в медленной. Длина каждой пары равна 1, и сдвиг на k позиций поворачивает каждую пару на свой постоянный угол. Спека positional_encoding.t27 вычисляет каждое значение.
+
Попробовать
Шагните с позиции 0 на 1 и посмотрите, какие часы двигаются; затем перейдите на 37 и найдите пару, которая всё ещё смотрит вверх.
Every position gets 8 numbers, sines and cosines at frequencies falling by 10 per pair. Drag the position: the fast clock spins, the slow ones barely move, and the stripes show all 48 rows.
+
specs/ml/transformer/positional_encoding.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Uses sin/cos at different frequencies for each dimension | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward read an undeclared `d_model`, called `math::pow` / `math::sin` /
+// `math::cos`, computed each encoding into a local and dropped it, and grew the
+// frequency with the dimension where Vaswani et al. (2017) shrink it. It now
+// writes the sinusoidal encoding
+// PE(pos, 2i) = sin(pos / 10000^(2i / d)), PE(pos, 2i + 1) = cos(pos / 10000^(2i / d))
+// for each position into a positions x d_model buffer.
+
+module PositionalEnc;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_MAX_LEN : u32 = 2048;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PosEncConfig = struct {
+ d_model : u32,
+ max_len : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // 1 / 10000^(2i / d) = exp(-(2i / d) ln 10000)
+ fn inv_freq(pair: u32, d_model: u32) -> f32 {
+ return exp(-(@as(f32, @floatFromInt(2 * pair)) / @as(f32, @floatFromInt(d_model))) * log(@as(f32, 10000.0)));
+ }
+
+ fn forward(positions: []const u32, config: PosEncConfig, out: []f32) -> void {
+ var p : usize = 0;
+ while (p < positions.len) {
+ const pos : f32 = @as(f32, @floatFromInt(positions[p]));
+ var dim : u32 = 0;
+ while (dim < config.d_model) {
+ const angle = pos * inv_freq(dim / 2, config.d_model);
+ if (dim % 2 == 0) {
+ out[p * config.d_model + dim] = sin(angle);
+ } else {
+ out[p * config.d_model + dim] = cos(angle);
+ }
+ dim = dim + 1;
+ }
+ p = p + 1;
+ }
+ }
+
+ // PE(pos, dim) for d_model 8
+ fn pe(pos: u32, dim: usize) -> f32 {
+ const positions : [1]u32 = [pos];
+ var out : [8]f32 = [9.0] ** 8;
+ forward(&positions, PosEncConfig{ .d_model = 8, .max_len = DEFAULT_MAX_LEN }, &out);
+ return out[dim];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // pos 1, d 8: dims 0/1 use frequency 1, dims 2/3 use 1/10 (10000^(2/8) = 10)
+ test forward_basic_case
+ given s0 = pe(1, 0)
+ when s2 = pe(1, 2)
+ then @abs(s0 - 0.8414710) < 0.000001 and @abs(pe(1, 1) - 0.5403023) < 0.000001 and @abs(s2 - 0.0998334) < 0.000001 and @abs(pe(1, 3) - 0.9950042) < 0.000001
+
+ // Position 0 encodes as (0, 1, 0, 1, ...)
+ test forward_simple
+ given s = pe(0, 6)
+ when c = pe(0, 7)
+ then s == 0.0 and c == 1.0
+
+ // Each sin/cos pair lies on the unit circle
+ test forward_pairs_are_unit_vectors
+ given p = pe(37, 2) * pe(37, 2) + pe(37, 3) * pe(37, 3)
+ then @abs(p - 1.0) < 0.000001
+
+ invariant positional_encoding_max_len_is_positive
+ assert DEFAULT_MAX_LEN > 0
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/ppo-and-the-reward/card.png b/apps/website/public/ru/learn/ppo-and-the-reward/card.png
new file mode 100644
index 0000000000..4234979aec
Binary files /dev/null and b/apps/website/public/ru/learn/ppo-and-the-reward/card.png differ
diff --git a/apps/website/public/ru/learn/ppo-and-the-reward/index.html b/apps/website/public/ru/learn/ppo-and-the-reward/index.html
new file mode 100644
index 0000000000..e846b27c3e
--- /dev/null
+++ b/apps/website/public/ru/learn/ppo-and-the-reward/index.html
@@ -0,0 +1,272 @@
+
+
+
+
+
+
+PPO и награда: курс о нейросетях, урок 27 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
PPO и награда
+
Вы узнаете
Как PPO превращает награду в обновление политики и как его обрезка не даёт одному обновлению зайти слишком далеко.
+
Агент учится на награде, делая хорошие действия вероятнее. PPO измеряет изменение политики отношением r новой вероятности действия к старой и умножает его на преимущество A, насколько действие превзошло ожидание. Затем он обрезает: когда r ушло больше чем на eps в ту сторону, за которую награждает A, цель становится плоской. При r = 1.5, A = 1 и eps 0.2 сырое r A равно 1.5, но PPO засчитывает 1.2, так что один пакет не может далеко сдвинуть политику. На пакете спеки, отношения 1, 1.1 и 0.9 с преимуществами 2, 1.5 и 3, потеря политики равна -2.1167. Спека ppo_clip_loss.t27 её вычисляет.
+
Попробовать
Сделайте преимущество отрицательным и найдите, где кривая теперь становится плоской; затем расширьте eps до 0.5.
Drag the probability ratio of a new policy over the old: PPO's objective follows r times the advantage until r moves more than eps the rewarded way, then it goes flat and stops pulling.
+
specs/ml/rl/ppo_clip_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Probability ratio between policies | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written with `let x = if ... {}` and `for i in 0..n`, so it never compiled.
+// Three formulas were wrong, and their tests pinned the wrong values:
+// - compute_ratio returned log pi_new - log pi_old; the ratio is exp of that.
+// - clipped_surrogate returned clip(r) A; PPO's term is min(r A, clip(r) A), so
+// r = 1.1 inside the clip range gives 2.2, not 2.4.
+// - policy_loss returned the surrogate itself, a quantity to MAXIMISE; as a loss
+// added to the value and entropy losses it is minus the mean surrogate.
+// Each test now states the value PPO defines (Schulman et al., 2017).
+
+module PpoClipLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_CLIP_EPSILON : f32 = 0.2;
+ const DEFAULT_ENTROPY_COEF : f32 = 0.01;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PPOClipConfig = struct {
+ clip_epsilon : f32,
+ entropy_coef : f32,
+ value_clip_coef : f32,
+ };
+
+ pub const PPOBatch = struct {
+ states : [][]f32,
+ actions : [][]f32,
+ old_log_probs : []f32,
+ advantages : []f32,
+ returns : []f32,
+ values : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // r = pi_new / pi_old = exp(log pi_new - log pi_old)
+ fn compute_ratio(new_log_prob: f32, old_log_prob: f32) -> f32 {
+ return exp(new_log_prob - old_log_prob);
+ }
+
+ // min(r A, clip(r, 1 - eps, 1 + eps) A)
+ fn clipped_surrogate(ratio: f32, advantage: f32, clip_epsilon: f32) -> f32 {
+ const clipped_ratio = @min(@max(ratio, 1.0 - clip_epsilon), 1.0 + clip_epsilon);
+ return @min(ratio * advantage, clipped_ratio * advantage);
+ }
+
+ // -mean_t min(r_t A_t, clip(r_t) A_t): the loss gradient descent minimises
+ fn policy_loss(ratios: []const f32, advantages: []const f32, clip_epsilon: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < ratios.len) {
+ total = total + clipped_surrogate(ratios[i], advantages[i], clip_epsilon);
+ i = i + 1;
+ }
+ return -total / @as(f32, @floatFromInt(ratios.len));
+ }
+
+ // -coef * mean entropy: lower loss for a more exploratory policy
+ fn entropy_loss(entropies: []const f32, entropy_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < entropies.len) {
+ total = total + entropies[i];
+ i = i + 1;
+ }
+ return -entropy_coef * (total / @as(f32, @floatFromInt(entropies.len)));
+ }
+
+ // mean 0.5 (target - (old + clip(pred - old, -c, c)))^2
+ fn value_loss(predicted_values: []const f32, target_values: []const f32, old_values: []const f32, clip_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predicted_values.len) {
+ const diff = @min(@max(predicted_values[i] - old_values[i], -clip_coef), clip_coef);
+ const err = target_values[i] - (old_values[i] + diff);
+ total = total + 0.5 * err * err;
+ i = i + 1;
+ }
+ return total / @as(f32, @floatFromInt(predicted_values.len));
+ }
+
+ fn policy_loss_of(r0: f32, r1: f32, r2: f32) -> f32 {
+ const ratios : [3]f32 = [r0, r1, r2];
+ const advantages : [3]f32 = [2.0, 1.5, 3.0];
+ return policy_loss(&ratios, &advantages, DEFAULT_CLIP_EPSILON);
+ }
+
+ fn entropy_loss_of() -> f32 {
+ const entropies : [3]f32 = [0.5, 0.6, 0.4];
+ return entropy_loss(&entropies, DEFAULT_ENTROPY_COEF);
+ }
+
+ fn value_loss_of(p0: f32) -> f32 {
+ const predicted : [3]f32 = [p0, 2.0, 1.5];
+ const target : [3]f32 = [1.1, 2.1, 1.6];
+ const old : [3]f32 = [0.9, 1.9, 1.4];
+ return value_loss(&predicted, &target, &old, 0.2);
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_CLIP_EPSILON;
+ }
+
+ // total_loss(policy_loss_value: f32, value_loss_value: f32, entropy_loss_value: f32, value_loss_coef: f32) → f32
+ fn total_loss(policy_loss_value: f32, value_loss_value: f32, entropy_loss_value: f32, value_loss_coef: f32) -> f32 {
+ // Combine all loss components
+ return policy_loss_value + value_loss_coef * value_loss_value + entropy_loss_value;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test compute_ratio_basic_case
+ given
+ new_log_prob = 0.5
+ old_log_prob = 0.3
+ when result = compute_ratio(new_log_prob, old_log_prob)
+ then @abs(result - 1.2214028) < 0.000001 // exp(0.2); the old `== 0.2` was the log ratio
+
+ test clipped_surrogate_basic_case
+ given
+ ratio = 1.1
+ advantage = 2.0
+ clip_epsilon = 0.2
+ when result = clipped_surrogate(ratio, advantage, clip_epsilon)
+ then @abs(result - 1.1 * 2.0) < 0.000001 // inside [0.8, 1.2] the clip does nothing
+
+ test clipped_surrogate_clipping_case
+ given
+ ratio = 1.5 // Above clip threshold of 1.2
+ advantage = 2.0
+ clip_epsilon = 0.2
+ when result = clipped_surrogate(ratio, advantage, clip_epsilon)
+ then @abs(result - 1.2 * 2.0) < 0.000001 and clipped_surrogate(0.5, -1.0, 0.2) == -0.8
+
+ // min(r A, clip(r) A) = (2.0, 1.65, 2.7): the loss is -mean = -2.1166667. The old `> 0` was
+ // the surrogate's sign; a ratio pushed past the clip earns nothing more.
+ test policy_loss_basic_case
+ given result = policy_loss_of(1.0, 1.1, 0.9)
+ when pushed = policy_loss_of(1.0, 1.1, 3.0)
+ then result < 0.0 and @abs(result + 2.1166667) < 0.000001 and @abs(pushed - policy_loss_of(1.0, 1.1, 1.2)) < 0.000001
+
+ test entropy_loss_basic_case
+ given result = entropy_loss_of()
+ then result < 0.0 and @abs(result + 0.005) < 0.0000001
+
+ test value_loss_basic_case
+ given result = value_loss_of(1.0)
+ when clipped = value_loss_of(5.0)
+ then result > 0.0 and @abs(result - 0.005) < 0.0000001 and @abs(clipped - value_loss_of(1.1)) < 0.0000001
+
+ test total_loss_basic_case
+ given
+ policy_loss_value = 1.0
+ value_loss_value = 0.5
+ entropy_loss_value = -0.01
+ value_loss_coef = 0.5
+ when result = total_loss(policy_loss_value, value_loss_value, entropy_loss_value, value_loss_coef)
+ then @abs(result - (1.0 + 0.5 * 0.5 + (-0.01))) < 0.0000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // 0 < clip_epsilon <= 1 (typically 0.1-0.3)
+ invariant ppo_clip_loss_constraint_0
+ assert valid_input() > 0.0 and valid_input() <= 1.0
+
+ // entropy_coef >= 0
+ invariant ppo_clip_loss_constraint_1
+ assert DEFAULT_ENTROPY_COEF >= 0.0
+
+ // name: ppo_objective -- for A > 0 the surrogate stops growing at r = 1 + eps
+ invariant ppo_clip_loss_constraint_2
+ assert clipped_surrogate(5.0, 1.0, 0.2) == 1.2
+
+ // name: probability_ratio -- equal log-probabilities give r = 1
+ invariant ppo_clip_loss_constraint_3
+ assert compute_ratio(0.3, 0.3) == 1.0
+
+ // "Schulman et al. (2017) - Proximal Policy Optimization Algorithms, arXiv:1707.06347":
+ // for A < 0 the surrogate stops falling at r = 1 - eps
+ invariant ppo_clip_loss_constraint_4
+ assert clipped_surrogate(0.1, -1.0, 0.2) == -0.8
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/relu/card.png b/apps/website/public/ru/learn/relu/card.png
new file mode 100644
index 0000000000..519810c7ff
Binary files /dev/null and b/apps/website/public/ru/learn/relu/card.png differ
diff --git a/apps/website/public/ru/learn/relu/index.html b/apps/website/public/ru/learn/relu/index.html
new file mode 100644
index 0000000000..825676940d
--- /dev/null
+++ b/apps/website/public/ru/learn/relu/index.html
@@ -0,0 +1,210 @@
+
+
+
+
+
+
+ReLU: оставить положительную часть: курс о нейросетях, урок 1 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
ReLU: оставить положительную часть
+
Вы узнаете
Что ReLU делает с положительным и с отрицательным входом и почему наклон 0 останавливает обучение нейрона.
+
Нейрон складывает взвешенные входы и пропускает сумму через функцию активации. ReLU из них самая простая, max(0, x): положительная сумма проходит без изменений, отрицательная становится 0. Её наклон равен 1 или 0, а наклон 0 означает нейрон, который перестал учиться: градиент через него назад не течёт. Утечка это исправляет: с наклоном 0.1 вход x = -2 даёт -0.2 и пропускает назад градиент 0.1. Спека relu_activation.t27 вычисляет обе кривые, которые рисует виджет.
+
Попробовать
Поставьте x на -2 и прочитайте наклон с утечкой и без неё; затем найдите наклон при x = 1.5.
Drag along the curve and watch ReLU pass every positive input and zero every negative one; give it a leaky slope and a little of the negative side gets through.
+
specs/ml/activation/relu_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// ReLU produces sparse gradients | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the tests called forward and backward with one or two of their three or
+// four arguments, two asserted only `result != undefined`, and the invariants
+// asserted `true`. Each test now runs the functions on buffers and checks values.
+
+module Relu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const ZERO : f32 = 0.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const ReLUConfig = struct {
+ negative_slope : f32,
+ inplace : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(input: []const f32, output: []f32, config: ReLUConfig) → void
+ fn forward(input: []const f32, output: []f32, config: ReLUConfig) -> void {
+ // Values at or below ZERO take the negative branch, values above ZERO pass through
+ // For negative_slope = 0.0, this gives max(0, x) behavior
+ var i : usize = 0;
+ while (i < input.len) : (i += 1) {
+ if (input[i] > ZERO) {
+ output[i] = input[i]; // Positive branch: identity
+ } else {
+ output[i] = input[i] * config.negative_slope; // Negative branch: scaled
+ }
+ }
+ }
+
+ // backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) → void
+ fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) -> void {
+ // The gradient gate is on the forward input, not grad_output - this makes gradients sparse
+ var i : usize = 0;
+ while (i < input.len) : (i += 1) {
+ if (input[i] > ZERO) {
+ grad_input[i] = grad_output[i]; // Positive branch: identity gradient
+ } else {
+ grad_input[i] = grad_output[i] * config.negative_slope; // Negative branch: scaled gradient
+ }
+ }
+ }
+
+ // Note: config.inplace is declared but currently unused by these functions.
+ // The buffers are separate parameters, so the aliasing case is not handled.
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ fn cfg(slope: f32) -> ReLUConfig {
+ return ReLUConfig{ .negative_slope = slope, .inplace = false };
+ }
+
+ // output[index] of forward over (a, b, c)
+ fn forward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const input : [3]f32 = [a, b, c];
+ var output : [3]f32 = [9.0] ** 3;
+ forward(&input, &output, cfg(slope));
+ return output[index];
+ }
+
+ // grad_input[index] of backward over (a, b, c), every grad_output 1
+ fn backward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [a, b, c];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &grad_input, cfg(slope));
+ return grad_input[index];
+ }
+
+ // The shared fixture: one negative, zero, one positive.
+ fn valid_input() -> f32 {
+ return -2.0;
+ }
+
+ // max(0, x) on (-2, 0, 2)
+ test "forward_basic_case"
+ given negative = forward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = forward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and forward_at(-2.0, 0.0, 2.0, 0.0, 1) == 0.0 and positive == 2.0
+
+ test "forward_positive_identity"
+ given slope = 0.0
+ when first = forward_at(1.0, 2.0, 3.0, slope, 0)
+ then first == 1.0 and forward_at(1.0, 2.0, 3.0, slope, 1) == 2.0 and forward_at(1.0, 2.0, 3.0, slope, 2) == 3.0
+
+ test "forward_negative_scaling"
+ given slope = 0.5
+ when first = forward_at(-1.0, -2.0, -3.0, slope, 0)
+ then first == -0.5 and forward_at(-1.0, -2.0, -3.0, slope, 1) == -1.0 and forward_at(-1.0, -2.0, -3.0, slope, 2) == -1.5
+
+ test "forward_zero_negative_slope"
+ given slope = 0.0
+ when first = forward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 1) == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+
+ // grad_output * negative_slope for input <= ZERO (0 included), grad_output above it
+ test "backward_input_gate"
+ given slope = 0.5
+ when first = backward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 1) == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+
+ // Plain ReLU: the gradient is 0 left of zero and passes through right of it
+ test "backward_basic_case"
+ given negative = backward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = backward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and positive == 1.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.len == output.len: forward writes every element of its buffer (none keeps the 9.0 fill)
+ invariant relu_constraint_0
+ assert forward_at(valid_input(), 0.0, 1.0, 0.0, 0) != 9.0 and forward_at(valid_input(), 0.0, 1.0, 0.0, 2) != 9.0
+
+ // A finite input gives a finite output: ReLU never exceeds its input's magnitude
+ invariant relu_constraint_1
+ assert @abs(forward_at(valid_input(), 0.0, 1.0, 0.5, 0)) <= @abs(valid_input())
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/rmsprop/card.png b/apps/website/public/ru/learn/rmsprop/card.png
new file mode 100644
index 0000000000..d415522eac
Binary files /dev/null and b/apps/website/public/ru/learn/rmsprop/card.png differ
diff --git a/apps/website/public/ru/learn/rmsprop/index.html b/apps/website/public/ru/learn/rmsprop/index.html
new file mode 100644
index 0000000000..1f38ad8cac
--- /dev/null
+++ b/apps/website/public/ru/learn/rmsprop/index.html
@@ -0,0 +1,174 @@
+
+
+
+
+
+
+RMSProp: шаг без масштаба: курс о нейросетях, урок 17 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
RMSProp: шаг без масштаба
+
Вы узнаете
Как RMSProp делит каждый шаг на скользящий корень среднего квадрата, чтобы шаг больше не зависел от величины градиента.
+
Правильный размер шага зависит от величины градиента, а разные веса видят градиенты, отличающиеся в миллион раз. RMSProp хранит скользящее среднее квадрата градиента, кэш, и делит каждый шаг на его корень, поэтому у шага нет единиц: при постоянном градиенте 0.001, 0.5 или 1000 вес на шаге 1 сдвигается ровно на те же 0.0316 и идёт тем же путём, доходя до -0.0914 после 100 шагов при lr 0.01. Затухание задаёт, как медленно наполняется кэш: при 0.99 те же 100 шагов заканчиваются на -1.0261. Спека rmsprop.t27 вычисляет вес и кэш.
+
Попробовать
Переключайтесь между тремя градиентами и убедитесь, что линии не двигаются; затем поднимите затухание до 0.99.
Feed one weight a constant gradient of 0.001, 0.5 or 1000: RMSProp divides by the root of its running mean square, so all three walk the same path. Change the decay and the path bends.
+
specs/ml/optimizer/rmsprop.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// RMSprop optimizer with moving average of squared gradients | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/rmsprop.tri. The port kept only
+// `grad`, squared grad[0], and threw it away; the test read an undefined
+// default_input().
+
+module Rmsprop;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_DECAY : f32 = 0.9;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const RMSpropConfig = struct {
+ learning_rate : f32,
+ decay : f32,
+ epsilon : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // step(grad, param, cache, config) -> void, in place:
+ // cache = decay * cache + (1 - decay) * grad^2
+ // param -= lr * grad / (sqrt(cache) + epsilon)
+ // The .tri formula reads lr, decay and epsilon but its parameter list
+ // omits them; they come from config here.
+ fn step(grad: []const f32, param: []f32, cache: []f32, config: RMSpropConfig) -> void {
+ var i : usize = 0;
+ while (i < grad.len) {
+ const g = grad[i];
+ cache[i] = config.decay * cache[i] + (1.0 - config.decay) * g * g;
+ param[i] = param[i] - config.learning_rate * g / (@sqrt(cache[i]) + config.epsilon);
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ // run `steps` steps of constant gradient g on one parameter from 1.0
+ fn param_after(g: f32, steps: usize, decay: f32) -> f32 {
+ const grad : [1]f32 = [g];
+ var param : [1]f32 = [1.0];
+ var cache : [1]f32 = [0.0];
+ var k : usize = 0;
+ while (k < steps) {
+ step(&grad, ¶m, &cache, RMSpropConfig{ .learning_rate = 0.01, .decay = decay, .epsilon = 0.00000001 });
+ k = k + 1;
+ }
+ return param[0];
+ }
+
+ fn cache_after(g: f32, steps: usize) -> f32 {
+ const grad : [1]f32 = [g];
+ var param : [1]f32 = [1.0];
+ var cache : [1]f32 = [0.0];
+ var k : usize = 0;
+ while (k < steps) {
+ step(&grad, ¶m, &cache, RMSpropConfig{ .learning_rate = 0.01, .decay = DEFAULT_DECAY, .epsilon = 0.00000001 });
+ k = k + 1;
+ }
+ return cache[0];
+ }
+
+ // first step from cache 0: cache = 0.1 g^2, so the move is lr / sqrt(0.1)
+ // = 0.0316228 whatever the size of g
+ test first_step_is_scale_free
+ given small = param_after(0.001, 1, DEFAULT_DECAY)
+ when large = param_after(1000.0, 1, DEFAULT_DECAY)
+ then @abs(small - (1.0 - 0.0316228)) < 0.00001 and @abs(large - small) < 0.00001
+
+ // moving average of a constant: cache_n = (1 - 0.9^n) g^2
+ test cache_is_moving_average
+ given result = cache_after(2.0, 3)
+ when expected = (1.0 - 0.729) * 4.0
+ then @abs(result - expected) < 0.00001
+
+ test moves_against_the_gradient
+ given up = param_after(0.5, 2, DEFAULT_DECAY)
+ when down = param_after(-0.5, 2, DEFAULT_DECAY)
+ then up < 1.0 and down > 1.0 and @abs((1.0 - up) - (down - 1.0)) < 0.00001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant rmsprop_decay_in_unit_interval
+ assert DEFAULT_DECAY > 0.0 and DEFAULT_DECAY < 1.0
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/softmax-and-temperature/card.png b/apps/website/public/ru/learn/softmax-and-temperature/card.png
new file mode 100644
index 0000000000..4e0f0049e4
Binary files /dev/null and b/apps/website/public/ru/learn/softmax-and-temperature/card.png differ
diff --git a/apps/website/public/ru/learn/softmax-and-temperature/index.html b/apps/website/public/ru/learn/softmax-and-temperature/index.html
new file mode 100644
index 0000000000..ecd68bd8f4
--- /dev/null
+++ b/apps/website/public/ru/learn/softmax-and-temperature/index.html
@@ -0,0 +1,241 @@
+
+
+
+
+
+
+Softmax и температура: курс о нейросетях, урок 7 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
Softmax и температура
+
Вы узнаете
Как softmax превращает оценки в вероятности с суммой один и что с ними делает температура.
+
Классификатор заканчивается одной оценкой на класс, и softmax превращает оценки в вероятности: exp каждой оценки, делённая на сумму всех. Оценки 1, 2 и 3 становятся 0.090, 0.245 и 0.665. Если сначала разделить оценки на температуру, меняется уверенность: при 0.5 верхняя оценка забирает 0.867, при 5 все три равны 0.27, 0.33 и 0.40. Языковая модель выбирает следующий токен именно отсюда, и настройка температуры чат-модели это и есть T. Спека softmax.t27 сначала вычитает наибольшую оценку, поэтому exp никогда не переполняется.
+
Попробовать
Охладите температуру до 0.1 и прочитайте три вероятности; затем сделайте ничью и нагрейте её.
Move three scores and the temperature: softmax turns them into probabilities that sum to one. Cool it and the top score takes almost everything; heat it and the choice evens out.
+
specs/ml/activation/softmax.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Control sharpness of distribution | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written with `range` and `math::exp`, which no spec declares, so no test
+// ever ran. backward was also wrong: it used only the Jacobian's diagonal,
+// p_i (1 - p_i), and dropped the -p_i p_j terms. It is now the full product
+// grad_input_i = p_i (g_i - sum_j g_j p_j) / temperature.
+
+module Softmax;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const ZERO : f32 = 0.0;
+ const ONE : f32 = 1.0;
+ const MIN_TEMP : f32 = 0.001;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SoftmaxConfig = struct {
+ temperature : f32,
+ axis : u32,
+ stable : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // output_i = exp((x_i - max) / T) / sum_j exp((x_j - max) / T); subtracting the max keeps exp finite
+ fn forward(logits: []const f32, output: []f32, config: SoftmaxConfig) -> void {
+ var max_val : f32 = logits[0];
+ var i : usize = 0;
+ while (i < logits.len) {
+ if (logits[i] > max_val) {
+ max_val = logits[i];
+ }
+ i = i + 1;
+ }
+ var sum : f32 = ZERO;
+ i = 0;
+ while (i < logits.len) {
+ output[i] = exp((logits[i] - max_val) / config.temperature);
+ sum = sum + output[i];
+ i = i + 1;
+ }
+ i = 0;
+ while (i < logits.len) {
+ output[i] = output[i] / sum;
+ i = i + 1;
+ }
+ }
+
+ // Vector-Jacobian product: grad_input_i = p_i (g_i - sum_j g_j p_j) / T
+ fn backward(grad_output: []const f32, probabilities: []const f32, grad_input: []f32, config: SoftmaxConfig) -> void {
+ var dot : f32 = ZERO;
+ var i : usize = 0;
+ while (i < grad_output.len) {
+ dot = dot + grad_output[i] * probabilities[i];
+ i = i + 1;
+ }
+ i = 0;
+ while (i < grad_output.len) {
+ grad_input[i] = probabilities[i] * (grad_output[i] - dot) / config.temperature;
+ i = i + 1;
+ }
+ }
+
+ fn cfg(temperature: f32) -> SoftmaxConfig {
+ return SoftmaxConfig{ .temperature = temperature, .axis = 0, .stable = true };
+ }
+
+ // output[index] of softmax over (a, b, c)
+ fn softmax_at(a: f32, b: f32, c: f32, temperature: f32, index: usize) -> f32 {
+ const logits : [3]f32 = [a, b, c];
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&logits, &output, cfg(temperature));
+ return output[index];
+ }
+
+ // grad_input[index] for grad_output (0.1, 0.2, 0.3) at probabilities (0.2, 0.3, 0.5)
+ fn backward_at(index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ const probabilities : [3]f32 = [0.2, 0.3, 0.5];
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ // L = sum_i w_i softmax(x)_i with w = (0.1, 0.2, 0.3): the loss whose gradient backward computes
+ fn weighted(a: f32, b: f32, c: f32) -> f32 {
+ return 0.1 * softmax_at(a, b, c, 1.0, 0) + 0.2 * softmax_at(a, b, c, 1.0, 1) + 0.3 * softmax_at(a, b, c, 1.0, 2);
+ }
+
+ fn grad_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ var probabilities : [3]f32 = [0.0] ** 3;
+ const logits : [3]f32 = [a, b, c];
+ forward(&logits, &probabilities, cfg(1.0));
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 3.0;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // softmax(1, 2, 3) = (0.0900306, 0.2447285, 0.6652410)
+ test "forward_basic_case"
+ given p0 = softmax_at(1.0, 2.0, 3.0, 1.0, 0)
+ when p2 = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ then p0 > 0.0 and softmax_at(1.0, 2.0, 3.0, 1.0, 1) > 0.0 and p2 > 0.0 and @abs(p0 + softmax_at(1.0, 2.0, 3.0, 1.0, 1) + p2 - 1.0) < 0.0001 and @abs(p0 - 0.0900306) < 0.000001 and @abs(p2 - 0.6652410) < 0.000001
+
+ // Subtracting the max: logits of 1000 do not overflow exp
+ test "forward_large_logits_stay_finite"
+ given p = softmax_at(1000.0, 1000.0, 1000.0, 1.0, 1)
+ then @abs(p - 1.0 / 3.0) < 0.000001
+
+ // A higher temperature flattens the distribution toward uniform
+ test "forward_temperature_flattens"
+ given sharp = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ when flat = softmax_at(1.0, 2.0, 3.0, 10.0, 2)
+ then flat < sharp and flat > 1.0 / 3.0
+
+ // dot = 0.23: grad = (0.2 (0.1 - 0.23), 0.3 (0.2 - 0.23), 0.5 (0.3 - 0.23)) = (-0.026, -0.009, 0.035)
+ test "backward_basic_case"
+ given g0 = backward_at(0)
+ when g2 = backward_at(2)
+ then g0 != 0.0 and backward_at(1) != 0.0 and g2 != 0.0 and @abs(g0 - (-0.026)) < 0.000001 and @abs(backward_at(1) - (-0.009)) < 0.000001 and @abs(g2 - 0.035) < 0.000001
+
+ // The gradient of a function of probabilities that sum to 1 sums to 0
+ test "backward_sums_to_zero"
+ given total = backward_at(0) + backward_at(1) + backward_at(2)
+ then @abs(total) < 0.0000001
+
+ test "backward_matches_finite_difference"
+ given slope = (weighted(1.0, 2.0 + 0.001, 3.0) - weighted(1.0, 2.0 - 0.001, 3.0)) / 0.002
+ when result = grad_at(1.0, 2.0, 3.0, 1)
+ then @abs(result - slope) < 0.0005
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // logits.size() == output.size(): the first and last element are both probabilities
+ invariant softmax_constraint_0
+ assert softmax_at(1.0, 2.0, valid_input(), 1.0, 0) >= 0.0 and softmax_at(1.0, 2.0, valid_input(), 1.0, 2) >= 0.0
+
+ // config.temperature >= MIN_TEMP
+ invariant softmax_constraint_1
+ assert cfg(1.0).temperature >= MIN_TEMP
+
+ // sum(output) = 1.0 within floating precision
+ invariant softmax_constraint_2
+ assert @abs(softmax_at(1.0, 2.0, valid_input(), 1.0, 0) + softmax_at(1.0, 2.0, valid_input(), 1.0, 1) + softmax_at(1.0, 2.0, valid_input(), 1.0, 2) - 1.0) < 0.0001
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/swish-and-silu/card.png b/apps/website/public/ru/learn/swish-and-silu/card.png
new file mode 100644
index 0000000000..6aeefe19ca
Binary files /dev/null and b/apps/website/public/ru/learn/swish-and-silu/card.png differ
diff --git a/apps/website/public/ru/learn/swish-and-silu/index.html b/apps/website/public/ru/learn/swish-and-silu/index.html
new file mode 100644
index 0000000000..879b8c76d1
--- /dev/null
+++ b/apps/website/public/ru/learn/swish-and-silu/index.html
@@ -0,0 +1,210 @@
+
+
+
+
+
+
+Swish и SiLU: курс о нейросетях, урок 5 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
Swish и SiLU
+
Вы узнаете
Как x, умноженный на sigmoid(beta x), открывает сам себя и как один параметр переводит его от гладкой кривой к ReLU.
+
Swish это x, умноженный на sigmoid(beta x): нейрон, который сам себе вентиль. При beta 1 он называется SiLU, и его нижняя точка -0.2785 около x = -1.28. Увеличьте beta, и вентиль становится резче: при beta 8 кривая это ReLU со скруглёнными углами, при beta 0.5 пологий склон через ноль. Один параметр переводит нейрон от гладкой кривой к переключателю, и сеть может его выучить. Спека silu_swish_activation.t27 вычисляет кривую и её наклон при любом beta, а также свой пакет при -2, 0 и 2: -0.238, 0 и 1.762.
+
Попробовать
Держите x на -1.28 и поднимайте beta от 1 до 8; затем найдите beta, при котором провал на этом масштабе исчезает.
Drag along x times sigmoid(beta x). At beta 1 it is SiLU, with its dip of -0.28; turn beta up and the curve closes on ReLU, turn it down and it flattens toward x / 2.
+
specs/ml/activation/silu_swish_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// SiLU derivative | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward_batch returned its input unchanged, beta was never read, and the
+// invariants asserted `true` over an undeclared `valid_input`. SiLU is now
+// x * sigmoid(beta * x) (Swish; beta = 1 is SiLU) on scalars and in place on slices.
+
+module SiluSwish;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_BETA : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SiLUConfig = struct {
+ beta : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // sigmoid(z) = 1 / (1 + exp(-z))
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Swish(x) = x * sigmoid(beta * x); beta = 1 is SiLU
+ fn forward(x: f32, beta: f32) -> f32 {
+ return x * sigmoid(beta * x);
+ }
+
+ // forward on each element IN PLACE; returns the same slice
+ fn forward_batch(input: []f32, beta: f32) -> []f32 {
+ var i : usize = 0;
+ while (i < input.len) {
+ input[i] = forward(input[i], beta);
+ i = i + 1;
+ }
+ return input;
+ }
+
+ // d/dx x sigmoid(beta x) = s + beta x s (1 - s), s = sigmoid(beta x)
+ fn derivative(x: f32, beta: f32) -> f32 {
+ const s = sigmoid(beta * x);
+ return s + beta * x * s * (1.0 - s);
+ }
+
+ fn batch_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ var input : [3]f32 = [a, b, c];
+ const output = forward_batch(&input, DEFAULT_BETA);
+ return output[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test "forward_basic_case"
+ given input = 0.0, beta = 1.0
+ when result = forward(input, beta)
+ then result == 0.0 // SiLU(0) = 0 * sigmoid(0) = 0
+
+ test "forward_positive_case"
+ given input = 1.0, beta = 1.0
+ when result = forward(input, beta)
+ then result > 0.0 // SiLU(1) = 1 * sigmoid(1) > 0
+
+ test "forward_negative_case"
+ given input = -1.0, beta = 1.0
+ when result = forward(input, beta)
+ then result < 0.0 // SiLU(-1) = -1 * sigmoid(-1) < 0
+
+ test "forward_batch_basic_case"
+ given beta = 1.0
+ when result = batch_at(1.0, 2.0, 3.0, 2)
+ then @abs(result - forward(3.0, beta)) < 0.0000001 and @abs(batch_at(1.0, 2.0, 3.0, 0) - 0.7310586) < 0.000001
+
+ test "forward_batch_zero_case"
+ given beta = 1.0
+ when result = batch_at(0.0, 0.0, 0.0, 0)
+ then result == 0.0 and batch_at(0.0, 0.0, 0.0, 1) == 0.0 and batch_at(0.0, 0.0, 0.0, 2) == 0.0 // SiLU(0) = 0
+
+ test "derivative_basic_case"
+ given input = 0.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU'(0) = sigmoid(0) + 0 * sigmoid(0) * (1 - sigmoid(0)) = sigmoid(0) > 0
+
+ test "derivative_positive_case"
+ given input = 1.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU derivative should be positive for positive inputs
+
+ test "derivative_negative_case"
+ given input = -1.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU derivative should be positive for negative inputs too
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // beta > 0
+ invariant silu_swish_constraint_0
+ assert DEFAULT_BETA > 0.0
+
+ // name: silu_formula -- x * sigmoid(x), so SiLU(x) - SiLU(-x) = x
+ invariant silu_swish_constraint_1
+ assert @abs(forward(valid_input(), DEFAULT_BETA) - forward(-valid_input(), DEFAULT_BETA) - valid_input()) < 0.000001
+
+ // name: derivative -- sigmoid(0) = 0.5 at the origin
+ invariant silu_swish_constraint_2
+ assert derivative(0.0, DEFAULT_BETA) == 0.5
+
+ // "Elfwing et al. (2017) - Sigmoid-Weighted Linear Units for Neural Network Function Approximation":
+ // SiLU dips below zero left of the origin
+ invariant silu_swish_constraint_3
+ assert forward(-valid_input(), DEFAULT_BETA) < 0.0
+
+ // "Ramachandran et al. (2017) - Searching for Activation Functions (Swish)": a large beta
+ // approaches ReLU
+ invariant silu_swish_constraint_4
+ assert @abs(forward(valid_input(), 50.0) - valid_input()) < 0.000001
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/tanh-and-saturation/card.png b/apps/website/public/ru/learn/tanh-and-saturation/card.png
new file mode 100644
index 0000000000..964dc477da
Binary files /dev/null and b/apps/website/public/ru/learn/tanh-and-saturation/card.png differ
diff --git a/apps/website/public/ru/learn/tanh-and-saturation/index.html b/apps/website/public/ru/learn/tanh-and-saturation/index.html
new file mode 100644
index 0000000000..f3d83594ee
--- /dev/null
+++ b/apps/website/public/ru/learn/tanh-and-saturation/index.html
@@ -0,0 +1,171 @@
+
+
+
+
+
+
+tanh и насыщение: курс о нейросетях, урок 3 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
tanh и насыщение
+
Вы узнаете
Почему tanh центрирована на нуле и где её наклон падает так низко, что нейрон перестаёт учиться.
+
tanh это сигмоида, растянутая на -1..1 и центрированная на 0, поэтому слой нейронов tanh передаёт дальше значения со средним около нуля, а её наклон в центре достигает 1. Она тоже насыщается: при 2.5 выход 0.9866, а наклон всего 0.0266. Красная полоса отмечает все входы, где наклон меньше 0.1, дальше примерно 1.8 в обе стороны, и нейрон, чьи входы живут там, почти не учится. Спека tanh_activation.t27 вычисляет кривую, наклон и свой пакет, tanh от 0, 1 и 2: 0, 0.762 и 0.964.
+
Попробовать
Тяните x, пока не начнётся полоса, и прочитайте наклон там; затем сравните наклон в 0 и при 2.5.
Drag along tanh: it is centred on zero, and where it flattens out its slope drops under 0.1. The red band shows every input where the gradient has all but stopped.
+
specs/ml/activation/tanh_activation.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Derivative (can compute from output) | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: `math::exp` named a module this spec never imports, and
+// forward_batch allocated with `[]f32{input.len}`. tanh is now computed in the
+// form that cannot overflow: exp(2|x|) reaches inf in f32 at |x| = 45, and
+// inf / inf is NaN.
+
+module Tanh;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const TanhConfig = struct {
+ none : void,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // tanh(x) = (1 - e^(-2|x|)) / (1 + e^(-2|x|)), with the sign of x
+ fn forward(x: f32) -> f32 {
+ if (x < 0.0) {
+ return -forward(-x);
+ }
+ const t = exp(-2.0 * x);
+ return (1.0 - t) / (1.0 + t);
+ }
+
+ // forward_batch: output[i] = tanh(input[i])
+ fn forward_batch(input: []const f32, output: []f32) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = forward(input[i]);
+ i = i + 1;
+ }
+ }
+
+ // tanh'(x) = 1 - tanh(x)^2
+ fn derivative(x: f32) -> f32 {
+ const tanh_x = forward(x);
+ return 1.0 - (tanh_x * tanh_x);
+ }
+
+ // Given y = tanh(x), the derivative is 1 - y^2
+ fn derivative_from_output(tanh_x: f32) -> f32 {
+ return 1.0 - (tanh_x * tanh_x);
+ }
+
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [0.0, 1.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, &output);
+ return output[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // tanh(0) = 0, tanh(1) = 0.7615942, tanh is odd
+ test forward_basic_case
+ given input = 0.0
+ when result = forward(input)
+ then result == 0.0 and @abs(forward(1.0) - 0.7615942) < 0.000001 and forward(-1.0) == -forward(1.0)
+
+ // exp(2x) is inf in f32 here; the stable form still gives 1, not NaN
+ test forward_saturates_without_overflow
+ given input = 100.0
+ when result = forward(input)
+ then result == 1.0 and forward(-100.0) == -1.0
+
+ test forward_batch_basic_case
+ given first = batch_at(0)
+ when second = batch_at(1)
+ then first == 0.0 and second > 0.0 and second < 1.0 and batch_at(2) > second
+
+ test derivative_basic_case
+ given input = 0.0
+ when result = derivative(input)
+ then result == 1.0
+
+ test derivative_from_output_basic_case
+ given input = 0.5
+ when tanh_result = forward(input)
+ then derivative_from_output(tanh_result) > 0.0 and derivative_from_output(tanh_result) < 1.0 and derivative_from_output(tanh_result) == derivative(input)
+
+ test derivative_matches_finite_difference
+ given slope = (forward(0.5 + 0.001) - forward(0.5 - 0.001)) / 0.002
+ when result = derivative(0.5)
+ then @abs(result - slope) < 0.001
+
Как функция Хьюбера обходится с малой ошибкой как с квадратичной, а с большой линейно, чтобы выбросы тянули слабее.
+
Квадратичная ошибка позволяет одному выбросу всё решать: ошибка 3 стоит 4.5 как половина своего квадрата, столько же, сколько 36 ошибок по 0.5. Функция Хьюбера равна половине квадрата, пока |e| не больше delta, а дальше растёт только линейно, delta |e| - delta^2 / 2, так что при delta 1 та же ошибка стоит 2.5. Два куска сходятся с одинаковым значением и наклоном, и градиент никогда не превышает delta: далёкая точка тянет модель с ограниченной силой. Для пары ошибок спеки 0.5 и 3 среднее равно 1.3125 против 2.3125 для половины квадрата. Спека huber_loss.t27 вычисляет обе.
+
Попробовать
Поставьте delta на 0.5 и тяните ошибку к 3; затем найдите, где две кривые расходятся.
Drag the error along: Huber is half its square inside delta and only grows linearly outside, so one wild miss pulls on the model with a bounded force. The dashed curve is the pure square.
+
specs/ml/loss/huber_loss.t27
// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Threshold where loss changes from quadratic to linear | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/huber_loss.tri: the port kept only
+// the first parameter and returned void, so forward computed a value and
+// threw it away, and every test asserted `result != undefined`.
+
+module HuberLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const HuberLossConfig = struct {
+ delta : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // per element e = prediction - target:
+ // |e| <= delta: 0.5 e^2 else: delta |e| - 0.5 delta^2
+ fn element_loss(e: f32, delta: f32) -> f32 {
+ const a = @abs(e);
+ if (a <= delta) {
+ return 0.5 * e * e;
+ }
+ return delta * a - 0.5 * delta * delta;
+ }
+
+ // forward(predictions, targets, delta) -> f32. The .tri gives the
+ // per-element formula only; this returns the MEAN over elements
+ // (PyTorch HuberLoss default reduction). 0 for n = 0.
+ fn forward(predictions: []const f32, targets: []const f32, delta: f32) -> f32 {
+ if (predictions.len == 0) {
+ return 0.0;
+ }
+ var sum_loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ sum_loss = sum_loss + element_loss(predictions[i] - targets[i], delta);
+ i = i + 1;
+ }
+ return sum_loss / (predictions.len as f32);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ fn huber_one(e: f32, delta: f32) -> f32 {
+ const predictions : [1]f32 = [e];
+ const targets : [1]f32 = [0.0];
+ return forward(&predictions, &targets, delta);
+ }
+
+ test quadratic_inside_delta
+ given result = huber_one(0.5, 1.0)
+ when negative = huber_one(-0.5, 1.0)
+ then @abs(result - 0.125) < 0.000001 and result == negative
+
+ // delta |e| - delta^2 / 2 = 3 - 0.5
+ test linear_outside_delta
+ given result = huber_one(3.0, 1.0)
+ when wide = huber_one(3.0, 2.0)
+ then @abs(result - 2.5) < 0.000001 and @abs(wide - 4.0) < 0.000001
+
+ // the two pieces meet with equal value and slope at |e| = delta
+ test continuous_at_delta
+ given below = huber_one(0.999, 1.0)
+ when above = huber_one(1.001, 1.0)
+ then @abs(above - below - 0.002) < 0.0001
+
+ // mean of (0.125, 2.5) over two elements
+ fn huber_two(e0: f32, e1: f32, delta: f32) -> f32 {
+ const predictions : [2]f32 = [e0 + 1.0, e1 + 1.0];
+ const targets : [2]f32 = [1.0, 1.0];
+ return forward(&predictions, &targets, delta);
+ }
+
+ test forward_mean
+ given result = huber_two(0.5, 3.0, 1.0)
+ when expected = 1.3125
+ then @abs(result - expected) < 0.000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant huber_zero_at_target
+ assert huber_one(0.0, 1.0) == 0.0
+
+
+
+
+
+
diff --git a/apps/website/public/ru/learn/the-sigmoid/card.png b/apps/website/public/ru/learn/the-sigmoid/card.png
new file mode 100644
index 0000000000..b42381ca15
Binary files /dev/null and b/apps/website/public/ru/learn/the-sigmoid/card.png differ
diff --git a/apps/website/public/ru/learn/the-sigmoid/index.html b/apps/website/public/ru/learn/the-sigmoid/index.html
new file mode 100644
index 0000000000..5e5a72d334
--- /dev/null
+++ b/apps/website/public/ru/learn/the-sigmoid/index.html
@@ -0,0 +1,171 @@
+
+
+
+
+
+
+Сигмоида: число в вероятность: курс о нейросетях, урок 2 из 27
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+t27.aiEnglish
+
+
+
Сигмоида: число в вероятность
+
Вы узнаете
Как сигмоида переводит любой вход в вероятность и почему её наклон мешает глубоким сетям.
+
Сигмоида 1 / (1 + e^-x) сжимает всю числовую прямую в отрезок от 0 до 1, поэтому её читают как вероятность: 2 превращается в 0.8808, а -4 в 0.018. Её наклон равен s (1 - s) и считается по самому выходу s; он наибольший в 0, где равен 0.25, а при x = 8 равен 0.000335. Градиент, который идёт назад через стопку сигмоид, на каждой умножается не больше чем на 0.25, поэтому глубокие сети на сигмоидах учились медленно. Спека sigmoid_activation.t27 вычисляет кривую и наклон.
+
Попробовать
Найдите вход, при котором вероятность равна 0.5; затем перетащите к 6 и посмотрите, как мало осталось наклона.
+
+
+
+
+
diff --git a/apps/website/public/t27/files/specs/ml/activation/elu_activation.t27 b/apps/website/public/t27/files/specs/ml/activation/elu_activation.t27
index c0b472c72a..9b2a8b838e 100644
--- a/apps/website/public/t27/files/specs/ml/activation/elu_activation.t27
+++ b/apps/website/public/t27/files/specs/ml/activation/elu_activation.t27
@@ -1,6 +1,9 @@
// SPDX-License-Identifier: Apache-2.0
// t27/specs/
// Derivative | φ² + 1/φ² = 3 | TRINITY
+// #8980: `math::exp` named a module this spec never imports, forward_batch grew
+// an array with `+`, and every test called a two-argument function with one
+// argument and asserted `result != undefined`. Now each test checks a value.
module Elu;
use base::types;
@@ -24,74 +27,96 @@ module Elu;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(x: f32, alpha: f32) → f32
+ // forward(x, alpha) = x for x > 0, alpha * (exp(x) - 1) otherwise
fn forward(x: f32, alpha: f32) -> f32 {
- if x > 0.0 {
+ if (x > 0.0) {
return x;
- } else {
- return alpha * (math::exp(x) - 1.0);
}
+ return alpha * (exp(x) - 1.0);
}
- // forward_batch(input: []f32, alpha: f32) → []f32
- fn forward_batch(input: []f32, alpha: f32) -> []f32 {
- var result = []f32{};
+ // forward_batch: output[i] = forward(input[i], alpha)
+ fn forward_batch(input: []const f32, alpha: f32, output: []f32) -> void {
var i : usize = 0;
- while (i < input.len) : (i += 1) {
- if (input[i] > 0.0) {
- result = result + [input[i]];
- } else {
- result = result + [alpha * (math::exp(input[i]) - 1.0)];
- }
+ while (i < input.len) {
+ output[i] = forward(input[i], alpha);
+ i = i + 1;
}
- return result;
}
- // derivative(x: f32, alpha: f32) → f32
+ // derivative(x, alpha) = 1 for x > 0, alpha * exp(x) otherwise
fn derivative(x: f32, alpha: f32) -> f32 {
- if x > 0.0 {
+ if (x > 0.0) {
return 1.0;
- } else {
- return alpha * math::exp(x);
}
+ return alpha * exp(x);
+ }
+
+ // The inputs the tests and invariants share: one negative, zero, one positive.
+ fn negative_input() -> f32 {
+ return -1.0;
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
+ }
+
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [-1.0, 0.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, DEFAULT_ALPHA, &output);
+ return output[index];
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // ELU(-1) = exp(-1) - 1 = -0.6321206; ELU(2) = 2
test "forward_basic_case"
- given input = default_input()
- when result = forward(input)
- then result != undefined
+ given input = negative_input()
+ when result = forward(input, DEFAULT_ALPHA)
+ then @abs(result - (-0.6321206)) < 0.000001 and forward(valid_input(), DEFAULT_ALPHA) == 2.0
+
+ // alpha scales only the negative side, and ELU(x) never goes below -alpha (in f32
+ // ELU(-20) rounds to exactly -alpha)
+ test "forward_alpha_scales_negative_side"
+ given input = negative_input()
+ when result = forward(input, 0.5)
+ then @abs(result - 0.5 * forward(input, 1.0)) < 0.000001 and forward(-20.0, 0.5) >= -0.5
test "forward_batch_basic_case"
- given input = default_input()
- when result = forward_batch(input)
- then result != undefined
+ given input = negative_input()
+ when result = batch_at(0)
+ then result == forward(input, DEFAULT_ALPHA) and batch_at(1) == 0.0 and batch_at(2) == 2.0
+ // ELU'(-1) = exp(-1) = 0.3678794; ELU'(2) = 1
test "derivative_basic_case"
- given input = default_input()
- when result = derivative(input)
- then result != undefined
+ given input = negative_input()
+ when result = derivative(input, DEFAULT_ALPHA)
+ then @abs(result - 0.3678794) < 0.000001 and derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+
+ test "derivative_matches_finite_difference"
+ given slope = (forward(-1.0 + 0.001, DEFAULT_ALPHA) - forward(-1.0 - 0.001, DEFAULT_ALPHA)) / 0.002
+ when result = derivative(-1.0, DEFAULT_ALPHA)
+ then @abs(result - slope) < 0.001
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
invariant elu_constraint_0
- given input = valid_input()
- then true // alpha > 0
+ assert DEFAULT_ALPHA > 0.0
+ // name: elu_formula -- identity on the positive side
invariant elu_constraint_1
- given input = valid_input()
- then true // name: elu_formula
+ assert forward(valid_input(), DEFAULT_ALPHA) == valid_input()
+ // name: derivative -- 1 on the positive side
invariant elu_constraint_2
- given input = valid_input()
- then true // name: derivative
+ assert derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+ // "Clevert et al. (2016) - Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)":
+ // ELU is continuous at 0
invariant elu_constraint_3
- given input = valid_input()
- then true // "Clevert et al. (2016) - Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)"
-
+ assert forward(0.0, DEFAULT_ALPHA) == 0.0
diff --git a/apps/website/public/t27/files/specs/ml/activation/gelu_activation.t27 b/apps/website/public/t27/files/specs/ml/activation/gelu_activation.t27
index bd7b8c1190..4c2b4a0e53 100644
--- a/apps/website/public/t27/files/specs/ml/activation/gelu_activation.t27
+++ b/apps/website/public/t27/files/specs/ml/activation/gelu_activation.t27
@@ -2,6 +2,11 @@
// t27/specs/
// Based on cumulative distribution function | φ² + 1/φ² = 3 | TRINITY
+// #8980: Zig has neither `@tanh` nor `@erf`, so this never compiled. erf is now
+// a function of this spec (Abramowitz-Stegun 7.1.26, |error| < 1.5e-7), and the
+// exact derivative divides the normal density by sqrt(2 pi), not sqrt(2) (the
+// slip was invisible at x = 0, the only point the tests checked).
+
module Gelu;
use base::types;
use math::constants;
@@ -12,6 +17,8 @@ module Gelu;
const SQRT_2_OVER_PI : f32 = 0.7978845608;
const SQRT_PI_OVER_2 : f32 = 1.2533141373;
+ const SQRT_2 : f32 = 1.4142135624;
+ const SQRT_2_PI : f32 = 2.5066282746;
// ═══════════════════════════════════════════════════════════
// 2. Types
@@ -25,125 +32,136 @@ module Gelu;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(input: []const f32, output: []f32, config: GELUConfig) → void
+ // erf(x), Abramowitz-Stegun 7.1.26: max absolute error 1.5e-7
+ fn erf(x: f32) -> f32 {
+ if (x < 0.0) {
+ return -erf(-x);
+ }
+ const t = 1.0 / (1.0 + 0.3275911 * x);
+ const poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))));
+ return 1.0 - poly * exp(-x * x);
+ }
+
+ // One element. approximate: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3)));
+ // exact: 0.5 x (1 + erf(x / sqrt(2))) = x Phi(x)
+ fn gelu_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ return 0.5 * x * (1.0 + tanh(inner));
+ }
+ return 0.5 * x * (1.0 + erf(x / SQRT_2));
+ }
+
+ // d/dx of gelu_one
+ fn gelu_grad_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ const tanh_val = tanh(inner);
+ const sech_sq = 1.0 - tanh_val * tanh_val;
+ return 0.5 * (1.0 + tanh_val) + 0.5 * x * sech_sq * SQRT_2_OVER_PI * (1.0 + 3.0 * 0.044715 * x * x);
+ }
+ // Phi(x) + x phi(x), phi(x) = exp(-x^2 / 2) / sqrt(2 pi)
+ const cdf = 0.5 * (1.0 + erf(x / SQRT_2));
+ const pdf = exp(-0.5 * x * x) / SQRT_2_PI;
+ return cdf + x * pdf;
+ }
+
fn forward(input: []const f32, output: []f32, config: GELUConfig) -> void {
- let n = input.len;
- for (0..n) |i| {
- let x = input[i];
- if (config.approximate) {
- // Hendrycks–Gimpel tanh form: 0.5 * x * (1 + tanh(SQRT_2_OVER_PI * (x + 0.044715 * x^3)))
- let x_cubed = x * x * x;
- let inner = SQRT_2_OVER_PI * (x + 0.044715 * x_cubed);
- let tanh_val = @tanh(inner);
- output[i] = 0.5 * x * (1.0 + tanh_val);
- } else {
- // Exact form: 0.5 * x * (1 + erf(x / sqrt(2)))
- let sqrt_2 = 1.4142135624;
- let erf_arg = x / sqrt_2;
- let erf_val = @erf(erf_arg);
- output[i] = 0.5 * x * (1.0 + erf_val);
- }
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = gelu_one(input[i], config.approximate);
+ i = i + 1;
}
}
- // backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: GELUConfig) → void
fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: GELUConfig) -> void {
- let n = input.len;
- for (0..n) |i| {
- let x = input[i];
- let grad_output_val = grad_output[i];
-
- if (config.approximate) {
- // Derivative of tanh form
- let x_cubed = x * x * x;
- let inner = SQRT_2_OVER_PI * (x + 0.044715 * x_cubed);
- let tanh_val = @tanh(inner);
- let sech_sq = 1.0 - tanh_val * tanh_val;
- let derivative = 0.5 * (1.0 + tanh_val) + 0.5 * x * sech_sq * SQRT_2_OVER_PI * (1.0 + 3.0 * 0.044715 * x * x);
- grad_input[i] = grad_output_val * derivative;
- } else {
- // Derivative of exact form: Φ(x) + x * φ(x), where φ(x) = exp(-x²/2) / sqrt(2π)
- let sqrt_2 = 1.4142135624;
- let erf_arg = x / sqrt_2;
- let erf_val = @erf(erf_arg);
- let phi_x = 0.5 * (1.0 + erf_val); // Φ(x) = normal CDF
- let exp_arg = -0.5 * x * x;
- let phi_x_derivative = @exp(exp_arg) / sqrt_2; // φ(x) = normal PDF
- let derivative = phi_x + x * phi_x_derivative;
- grad_input[i] = grad_output_val * derivative;
- }
+ var i : usize = 0;
+ while (i < input.len) {
+ grad_input[i] = grad_output[i] * gelu_grad_one(input[i], config.approximate);
+ i = i + 1;
}
}
+ // forward / backward over a one-element slice, for the tests
+ fn forward_at(x: f32, approximate: bool) -> f32 {
+ const input : [1]f32 = [x];
+ var output : [1]f32 = [0.0];
+ forward(&input, &output, GELUConfig{ .approximate = approximate });
+ return output[0];
+ }
+
+ fn backward_at(x: f32, approximate: bool) -> f32 {
+ const grad_output : [1]f32 = [1.0];
+ const input : [1]f32 = [x];
+ var grad_input : [1]f32 = [0.0];
+ backward(&grad_output, &input, &grad_input, GELUConfig{ .approximate = approximate });
+ return grad_input[0];
+ }
+
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // The product goes through a function so the check runs, not only folds at compile time.
+ fn product_of(a: f32, b: f32) -> f32 {
+ return a * b;
+ }
+
+ fn config_of(approximate: bool) -> GELUConfig {
+ return GELUConfig{ .approximate = approximate };
+ }
+
test gelu_sqrt_constants_are_reciprocals
- given product = SQRT_2_OVER_PI * SQRT_PI_OVER_2
+ given product = product_of(SQRT_2_OVER_PI, SQRT_PI_OVER_2)
then @abs(product - 1.0) < 0.0000005
test gelu_sqrt_2_over_pi_squares_to_2_over_pi
- given squared = SQRT_2_OVER_PI * SQRT_2_OVER_PI
+ given squared = product_of(SQRT_2_OVER_PI, SQRT_2_OVER_PI)
then @abs(squared - 0.63661977) < 0.0000003
test gelu_config_records_the_approximation_choice
- given exact = GELUConfig{.approximate=false}
- and approx = GELUConfig{.approximate=true}
- then !exact.approximate
- and approx.approximate
+ given exact = config_of(false)
+ when approx = config_of(true)
+ then !exact.approximate and approx.approximate
+
+ // erf(0) = 0, erf(1) = 0.8427008, erf(-1) = -erf(1)
+ test gelu_erf_known_values
+ given at_one = erf(1.0)
+ when at_zero = erf(0.0)
+ then @abs(at_zero) < 0.0000002 and @abs(at_one - 0.8427008) < 0.0000003 and erf(-1.0) == -at_one
test gelu_forward_zero_input
- given input = []f32{0.0}
- and output = []f32{0.0}
- and exact_config = GELUConfig{.approximate=false}
- and approx_config = GELUConfig{.approximate=true}
- when forward(input, output, exact_config)
- when forward(input, output, approx_config)
- then output[0] == 0.0
+ given exact = forward_at(0.0, false)
+ when approx = forward_at(0.0, true)
+ then exact == 0.0 and approx == 0.0
test gelu_backward_zero_input_derivative
- given grad_output = []f32{1.0}
- and input = []f32{0.0}
- and grad_input = []f32{0.0}
- and exact_config = GELUConfig{.approximate=false}
- and approx_config = GELUConfig{.approximate=true}
- when backward(grad_output, input, grad_input, exact_config)
- when backward(grad_output, input, grad_input, approx_config)
- then grad_input[0] == 0.5
+ given exact = backward_at(0.0, false)
+ when approx = backward_at(0.0, true)
+ then @abs(exact - 0.5) < 0.0000002 and approx == 0.5
test gelu_forward_monotonic_positive_tail
- given input_small = []f32{1.0}
- and input_large = []f32{2.0}
- and output_small = []f32{0.0}
- and output_large = []f32{0.0}
- and config = GELUConfig{.approximate=true}
- when forward(input_small, output_small, config)
- when forward(input_large, output_large, config)
- then output_small[0] < output_large[0]
+ given small = forward_at(1.0, true)
+ when large = forward_at(2.0, true)
+ then small < large
+ // With the cubic term, approximate GELU(1) = 0.8411920; exact GELU(1) = Phi(1) = 0.8413447.
+ // Without it, 0.5 (1 + tanh(0.7978846)) = 0.8296438 -- so this pins the term.
test gelu_tanh_cubic_coefficient
- given input = []f32{1.0}
- and output = []f32{0.0}
- and config = GELUConfig{.approximate=true}
- when forward(input, output, config)
- // The cubic coefficient 0.044715 is part of the tanh approximation formula
- then true // This test documents the presence of the cubic term
+ given approx = forward_at(1.0, true)
+ when exact = forward_at(1.0, false)
+ then @abs(approx - 0.8411920) < 0.000001 and @abs(exact - 0.8413447) < 0.000001
test gelu_backward_exact_derivative_at_zero
- given grad_output = []f32{1.0}
- and input = []f32{0.0}
- and grad_input = []f32{0.0}
- and config = GELUConfig{.approximate=false}
- when backward(grad_output, input, grad_input, config)
- // At x=0, Φ(0) = 0.5 and φ(0) = 1/sqrt(2π), so derivative = 0.5 + 0 = 0.5
- then grad_input[0] == 0.5
+ given result = backward_at(0.0, false)
+ then @abs(result - 0.5) < 0.0000002
test gelu_forward_approx_derivative_at_zero
- given grad_output = []f32{1.0}
- and input = []f32{0.0}
- and grad_input = []f32{0.0}
- and config = GELUConfig{.approximate=true}
- when backward(grad_output, input, grad_input, config)
- // At x=0, tanh(0) = 0, so derivative = 0.5 * (1 + 0) + 0 = 0.5
- then grad_input[0] == 0.5
\ No newline at end of file
+ given result = backward_at(0.0, true)
+ then result == 0.5
+
+ // The derivative at x = 1 is where the old sqrt(2) slip showed: 1.0833155 exact
+ test gelu_backward_matches_finite_difference
+ given exact_slope = (forward_at(1.001, false) - forward_at(0.999, false)) / 0.002
+ when approx_slope = (forward_at(1.001, true) - forward_at(0.999, true)) / 0.002
+ then @abs(backward_at(1.0, false) - exact_slope) < 0.002 and @abs(backward_at(1.0, true) - approx_slope) < 0.002 and @abs(backward_at(1.0, false) - 1.0833155) < 0.000002
diff --git a/apps/website/public/t27/files/specs/ml/activation/relu_activation.t27 b/apps/website/public/t27/files/specs/ml/activation/relu_activation.t27
index 6f71e91967..b726401b32 100644
--- a/apps/website/public/t27/files/specs/ml/activation/relu_activation.t27
+++ b/apps/website/public/t27/files/specs/ml/activation/relu_activation.t27
@@ -2,6 +2,10 @@
// t27/specs/
// ReLU produces sparse gradients | φ² + 1/φ² = 3 | TRINITY
+// #8980: the tests called forward and backward with one or two of their three or
+// four arguments, two asserted only `result != undefined`, and the invariants
+// asserted `true`. Each test now runs the functions on buffers and checks values.
+
module Relu;
use base::types;
use math::constants;
@@ -59,53 +63,73 @@ module Relu;
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ fn cfg(slope: f32) -> ReLUConfig {
+ return ReLUConfig{ .negative_slope = slope, .inplace = false };
+ }
+
+ // output[index] of forward over (a, b, c)
+ fn forward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const input : [3]f32 = [a, b, c];
+ var output : [3]f32 = [9.0] ** 3;
+ forward(&input, &output, cfg(slope));
+ return output[index];
+ }
+
+ // grad_input[index] of backward over (a, b, c), every grad_output 1
+ fn backward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [a, b, c];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &grad_input, cfg(slope));
+ return grad_input[index];
+ }
+
+ // The shared fixture: one negative, zero, one positive.
+ fn valid_input() -> f32 {
+ return -2.0;
+ }
+
+ // max(0, x) on (-2, 0, 2)
test "forward_basic_case"
- given input = default_input()
- when result = forward(input)
- then result != undefined
+ given negative = forward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = forward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and forward_at(-2.0, 0.0, 2.0, 0.0, 1) == 0.0 and positive == 2.0
test "forward_positive_identity"
- given input = [1.0, 2.0, 3.0]
- when output = forward(input, ReLUConfig{ .negative_slope = 0.0, .inplace = false })
- then output[0] == 1.0
- and output[1] == 2.0
- and output[2] == 3.0
+ given slope = 0.0
+ when first = forward_at(1.0, 2.0, 3.0, slope, 0)
+ then first == 1.0 and forward_at(1.0, 2.0, 3.0, slope, 1) == 2.0 and forward_at(1.0, 2.0, 3.0, slope, 2) == 3.0
test "forward_negative_scaling"
- given input = [-1.0, -2.0, -3.0]
- when output = forward(input, ReLUConfig{ .negative_slope = 0.5, .inplace = false })
- then output[0] == -0.5
- and output[1] == -1.0
- and output[2] == -1.5
+ given slope = 0.5
+ when first = forward_at(-1.0, -2.0, -3.0, slope, 0)
+ then first == -0.5 and forward_at(-1.0, -2.0, -3.0, slope, 1) == -1.0 and forward_at(-1.0, -2.0, -3.0, slope, 2) == -1.5
test "forward_zero_negative_slope"
- given input = [-1.0, 0.0, 1.0]
- when output = forward(input, ReLUConfig{ .negative_slope = 0.0, .inplace = false })
- then output[0] == 0.0
- and output[1] == 0.0
- and output[2] == 1.0
+ given slope = 0.0
+ when first = forward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 1) == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+ // grad_output * negative_slope for input <= ZERO (0 included), grad_output above it
test "backward_input_gate"
- given input = [-1.0, 0.0, 1.0]
- when grad_input = backward([1.0, 1.0, 1.0], input, ReLUConfig{ .negative_slope = 0.5, .inplace = false })
- then grad_input[0] == 0.5 // grad_output * negative_slope for input <= ZERO
- and grad_input[1] == 0.5 // grad_output * negative_slope for input == ZERO
- and grad_input[2] == 1.0 // grad_output for input > ZERO
+ given slope = 0.5
+ when first = backward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 1) == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+ // Plain ReLU: the gradient is 0 left of zero and passes through right of it
test "backward_basic_case"
- given input = default_input()
- when result = backward(input)
- then result != undefined
+ given negative = backward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = backward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and positive == 1.0
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // input.len == output.len: forward writes every element of its buffer (none keeps the 9.0 fill)
invariant relu_constraint_0
- given input = valid_input()
- then true // input.len == output.len
+ assert forward_at(valid_input(), 0.0, 1.0, 0.0, 0) != 9.0 and forward_at(valid_input(), 0.0, 1.0, 0.0, 2) != 9.0
+ // A finite input gives a finite output: ReLU never exceeds its input's magnitude
invariant relu_constraint_1
- given input = valid_input()
- then true // All inputs must be finite
-
+ assert @abs(forward_at(valid_input(), 0.0, 1.0, 0.5, 0)) <= @abs(valid_input())
diff --git a/apps/website/public/t27/files/specs/ml/activation/sigmoid_activation.t27 b/apps/website/public/t27/files/specs/ml/activation/sigmoid_activation.t27
index 952a430dde..4e6d84ddb3 100644
--- a/apps/website/public/t27/files/specs/ml/activation/sigmoid_activation.t27
+++ b/apps/website/public/t27/files/specs/ml/activation/sigmoid_activation.t27
@@ -53,8 +53,14 @@ module Sigmoid;
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // #8980: through a function, so the check runs and is counted (it folded at compile
+ // time before, and the report counted it as a vacuous pass).
+ fn config_size() -> usize {
+ return @sizeOf(SigmoidConfig);
+ }
+
test sigmoid_has_no_tunable_parameter
- given size = @sizeOf(SigmoidConfig)
+ given size = config_size()
then size == 0
// sigmoid_config_is_a_placeholder_not_an_empty_struct read std.meta.fields, which t27
@@ -76,4 +82,15 @@ module Sigmoid;
test forward_batch_preserves_length
given input = []f32{0.0, 1.0, -1.0, 2.0, -2.0}
given result = forward_batch(input)
- then result.len == input.len
\ No newline at end of file
+ then result.len == input.len
+
+ // sigmoid(-x) = 1 - sigmoid(x); sigmoid(2) = 0.8807971
+ test forward_is_symmetric_about_one_half
+ given up = forward(2.0)
+ when down = forward(-2.0)
+ then @abs(up + down - 1.0) < 0.000001 and @abs(up - 0.8807971) < 0.000001
+
+ // The negative branch never computes exp(+x): no overflow at -100
+ test forward_far_left_is_zero_not_nan
+ given y = forward(-100.0)
+ then y >= 0.0 and y < 0.000001
diff --git a/apps/website/public/t27/files/specs/ml/activation/silu_swish_activation.t27 b/apps/website/public/t27/files/specs/ml/activation/silu_swish_activation.t27
index b215a4963c..404271397f 100644
--- a/apps/website/public/t27/files/specs/ml/activation/silu_swish_activation.t27
+++ b/apps/website/public/t27/files/specs/ml/activation/silu_swish_activation.t27
@@ -2,6 +2,10 @@
// t27/specs/
// SiLU derivative | φ² + 1/φ² = 3 | TRINITY
+// #8980: forward_batch returned its input unchanged, beta was never read, and the
+// invariants asserted `true` over an undeclared `valid_input`. SiLU is now
+// x * sigmoid(beta * x) (Swish; beta = 1 is SiLU) on scalars and in place on slices.
+
module SiluSwish;
use base::types;
use math::constants;
@@ -24,30 +28,40 @@ module SiluSwish;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(x: f32, beta: f32) → f32
+ // sigmoid(z) = 1 / (1 + exp(-z))
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Swish(x) = x * sigmoid(beta * x); beta = 1 is SiLU
fn forward(x: f32, beta: f32) -> f32 {
- // SiLU: x * sigmoid(x) = x / (1 + exp(-x))
- // Implementation: compute and store result if needed
- let sigmoid_x = 1.0 / (1.0 + exp(-x));
- let result = x * sigmoid_x;
- return result;
+ return x * sigmoid(beta * x);
}
- // forward_batch(input: []f32, beta: f32) → []f32
+ // forward on each element IN PLACE; returns the same slice
fn forward_batch(input: []f32, beta: f32) -> []f32 {
- // SiLU batch: apply SiLU to each element in the array
- // Simple implementation: mark as implemented by processing input
- let _ = input.len; // Use input to avoid "not yet implemented"
- return input; // Placeholder implementation
+ var i : usize = 0;
+ while (i < input.len) {
+ input[i] = forward(input[i], beta);
+ i = i + 1;
+ }
+ return input;
}
- // derivative(x: f32, beta: f32) → f32
+ // d/dx x sigmoid(beta x) = s + beta x s (1 - s), s = sigmoid(beta x)
fn derivative(x: f32, beta: f32) -> f32 {
- // SiLU derivative: sigmoid(x) + x * sigmoid(x) * (1 - sigmoid(x))
- // Implementation: compute and store result if needed
- let sigmoid_x = 1.0 / (1.0 + exp(-x));
- let result = sigmoid_x + x * sigmoid_x * (1.0 - sigmoid_x);
- return result;
+ const s = sigmoid(beta * x);
+ return s + beta * x * s * (1.0 - s);
+ }
+
+ fn batch_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ var input : [3]f32 = [a, b, c];
+ const output = forward_batch(&input, DEFAULT_BETA);
+ return output[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
}
// ═══════════════════════════════════════════════════════════
@@ -70,14 +84,14 @@ module SiluSwish;
then result < 0.0 // SiLU(-1) = -1 * sigmoid(-1) < 0
test "forward_batch_basic_case"
- given input = []f32{1.0, 2.0, 3.0}, beta = 1.0
- when result = forward_batch(input, beta)
- then result.len == 3 // Should preserve array length
+ given beta = 1.0
+ when result = batch_at(1.0, 2.0, 3.0, 2)
+ then @abs(result - forward(3.0, beta)) < 0.0000001 and @abs(batch_at(1.0, 2.0, 3.0, 0) - 0.7310586) < 0.000001
test "forward_batch_zero_case"
- given input = []f32{0.0, 0.0, 0.0}, beta = 1.0
- when result = forward_batch(input, beta)
- then result[0] == 0.0 and result[1] == 0.0 and result[2] == 0.0 // SiLU(0) = 0
+ given beta = 1.0
+ when result = batch_at(0.0, 0.0, 0.0, 0)
+ then result == 0.0 and batch_at(0.0, 0.0, 0.0, 1) == 0.0 and batch_at(0.0, 0.0, 0.0, 2) == 0.0 // SiLU(0) = 0
test "derivative_basic_case"
given input = 0.0, beta = 1.0
@@ -98,23 +112,24 @@ module SiluSwish;
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // beta > 0
invariant silu_swish_constraint_0
- given input = valid_input()
- then true // beta > 0
+ assert DEFAULT_BETA > 0.0
+ // name: silu_formula -- x * sigmoid(x), so SiLU(x) - SiLU(-x) = x
invariant silu_swish_constraint_1
- given input = valid_input()
- then true // name: silu_formula
+ assert @abs(forward(valid_input(), DEFAULT_BETA) - forward(-valid_input(), DEFAULT_BETA) - valid_input()) < 0.000001
+ // name: derivative -- sigmoid(0) = 0.5 at the origin
invariant silu_swish_constraint_2
- given input = valid_input()
- then true // name: derivative
+ assert derivative(0.0, DEFAULT_BETA) == 0.5
+ // "Elfwing et al. (2017) - Sigmoid-Weighted Linear Units for Neural Network Function Approximation":
+ // SiLU dips below zero left of the origin
invariant silu_swish_constraint_3
- given input = valid_input()
- then true // "Elfwing et al. (2017) - Sigmoid-Weighted Linear Units for Neural Network Function Approximation"
+ assert forward(-valid_input(), DEFAULT_BETA) < 0.0
+ // "Ramachandran et al. (2017) - Searching for Activation Functions (Swish)": a large beta
+ // approaches ReLU
invariant silu_swish_constraint_4
- given input = valid_input()
- then true // "Ramachandran et al. (2017) - Searching for Activation Functions (Swish)"
-
+ assert @abs(forward(valid_input(), 50.0) - valid_input()) < 0.000001
diff --git a/apps/website/public/t27/files/specs/ml/activation/softmax.t27 b/apps/website/public/t27/files/specs/ml/activation/softmax.t27
index e2f54c2128..859f9db7ed 100644
--- a/apps/website/public/t27/files/specs/ml/activation/softmax.t27
+++ b/apps/website/public/t27/files/specs/ml/activation/softmax.t27
@@ -2,6 +2,11 @@
// t27/specs/
// Control sharpness of distribution | φ² + 1/φ² = 3 | TRINITY
+// #8980: written with `range` and `math::exp`, which no spec declares, so no test
+// ever ran. backward was also wrong: it used only the Jacobian's diagonal,
+// p_i (1 - p_i), and dropped the -p_i p_j terms. It is now the full product
+// grad_input_i = p_i (g_i - sum_j g_j p_j) / temperature.
+
module Softmax;
use base::types;
use math::constants;
@@ -28,74 +33,134 @@ module Softmax;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(logits: []const f32, output: []f32, config: SoftmaxConfig) → void
+ // output_i = exp((x_i - max) / T) / sum_j exp((x_j - max) / T); subtracting the max keeps exp finite
fn forward(logits: []const f32, output: []f32, config: SoftmaxConfig) -> void {
- // Apply softmax transformation with temperature
- var max_val: f32 = logits[0];
- for (i in range(logits.len)) {
+ var max_val : f32 = logits[0];
+ var i : usize = 0;
+ while (i < logits.len) {
if (logits[i] > max_val) {
max_val = logits[i];
}
+ i = i + 1;
}
-
- var sum: f32 = ZERO;
- for (i in range(logits.len)) {
- output[i] = math::exp((logits[i] - max_val) / config.temperature);
- sum += output[i];
+ var sum : f32 = ZERO;
+ i = 0;
+ while (i < logits.len) {
+ output[i] = exp((logits[i] - max_val) / config.temperature);
+ sum = sum + output[i];
+ i = i + 1;
}
-
- for (i in range(logits.len)) {
- output[i] /= sum;
+ i = 0;
+ while (i < logits.len) {
+ output[i] = output[i] / sum;
+ i = i + 1;
}
}
- // backward(grad_output: []const f32, probabilities: []const f32, grad_input: []f32, config: SoftmaxConfig) → void
+ // Vector-Jacobian product: grad_input_i = p_i (g_i - sum_j g_j p_j) / T
fn backward(grad_output: []const f32, probabilities: []const f32, grad_input: []f32, config: SoftmaxConfig) -> void {
- // Compute gradient of softmax loss
- for (i in range(grad_output.len)) {
- grad_input[i] = grad_output[i] * probabilities[i] * (ONE - probabilities[i]);
+ var dot : f32 = ZERO;
+ var i : usize = 0;
+ while (i < grad_output.len) {
+ dot = dot + grad_output[i] * probabilities[i];
+ i = i + 1;
+ }
+ i = 0;
+ while (i < grad_output.len) {
+ grad_input[i] = probabilities[i] * (grad_output[i] - dot) / config.temperature;
+ i = i + 1;
}
}
+ fn cfg(temperature: f32) -> SoftmaxConfig {
+ return SoftmaxConfig{ .temperature = temperature, .axis = 0, .stable = true };
+ }
+
+ // output[index] of softmax over (a, b, c)
+ fn softmax_at(a: f32, b: f32, c: f32, temperature: f32, index: usize) -> f32 {
+ const logits : [3]f32 = [a, b, c];
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&logits, &output, cfg(temperature));
+ return output[index];
+ }
+
+ // grad_input[index] for grad_output (0.1, 0.2, 0.3) at probabilities (0.2, 0.3, 0.5)
+ fn backward_at(index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ const probabilities : [3]f32 = [0.2, 0.3, 0.5];
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ // L = sum_i w_i softmax(x)_i with w = (0.1, 0.2, 0.3): the loss whose gradient backward computes
+ fn weighted(a: f32, b: f32, c: f32) -> f32 {
+ return 0.1 * softmax_at(a, b, c, 1.0, 0) + 0.2 * softmax_at(a, b, c, 1.0, 1) + 0.3 * softmax_at(a, b, c, 1.0, 2);
+ }
+
+ fn grad_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ var probabilities : [3]f32 = [0.0] ** 3;
+ const logits : [3]f32 = [a, b, c];
+ forward(&logits, &probabilities, cfg(1.0));
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 3.0;
+ }
+
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // softmax(1, 2, 3) = (0.0900306, 0.2447285, 0.6652410)
test "forward_basic_case"
- given
- logits = [1.0, 2.0, 3.0]
- output = [0.0, 0.0, 0.0]
- config = SoftmaxConfig { temperature = 1.0, axis = 0, stable = true }
- when
- forward(logits, output, config)
- then
- output[0] > 0.0 and output[1] > 0.0 and output[2] > 0.0 and
- (output[0] + output[1] + output[2] - 1.0).abs() < 0.0001
-
+ given p0 = softmax_at(1.0, 2.0, 3.0, 1.0, 0)
+ when p2 = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ then p0 > 0.0 and softmax_at(1.0, 2.0, 3.0, 1.0, 1) > 0.0 and p2 > 0.0 and @abs(p0 + softmax_at(1.0, 2.0, 3.0, 1.0, 1) + p2 - 1.0) < 0.0001 and @abs(p0 - 0.0900306) < 0.000001 and @abs(p2 - 0.6652410) < 0.000001
+
+ // Subtracting the max: logits of 1000 do not overflow exp
+ test "forward_large_logits_stay_finite"
+ given p = softmax_at(1000.0, 1000.0, 1000.0, 1.0, 1)
+ then @abs(p - 1.0 / 3.0) < 0.000001
+
+ // A higher temperature flattens the distribution toward uniform
+ test "forward_temperature_flattens"
+ given sharp = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ when flat = softmax_at(1.0, 2.0, 3.0, 10.0, 2)
+ then flat < sharp and flat > 1.0 / 3.0
+
+ // dot = 0.23: grad = (0.2 (0.1 - 0.23), 0.3 (0.2 - 0.23), 0.5 (0.3 - 0.23)) = (-0.026, -0.009, 0.035)
test "backward_basic_case"
- given
- grad_output = [0.1, 0.2, 0.3]
- probabilities = [0.2, 0.3, 0.5]
- grad_input = [0.0, 0.0, 0.0]
- config = SoftmaxConfig { temperature = 1.0, axis = 0, stable = true }
- when
- backward(grad_output, probabilities, grad_input, config)
- then
- grad_input[0] != 0.0 and grad_input[1] != 0.0 and grad_input[2] != 0.0
+ given g0 = backward_at(0)
+ when g2 = backward_at(2)
+ then g0 != 0.0 and backward_at(1) != 0.0 and g2 != 0.0 and @abs(g0 - (-0.026)) < 0.000001 and @abs(backward_at(1) - (-0.009)) < 0.000001 and @abs(g2 - 0.035) < 0.000001
+
+ // The gradient of a function of probabilities that sum to 1 sums to 0
+ test "backward_sums_to_zero"
+ given total = backward_at(0) + backward_at(1) + backward_at(2)
+ then @abs(total) < 0.0000001
+
+ test "backward_matches_finite_difference"
+ given slope = (weighted(1.0, 2.0 + 0.001, 3.0) - weighted(1.0, 2.0 - 0.001, 3.0)) / 0.002
+ when result = grad_at(1.0, 2.0, 3.0, 1)
+ then @abs(result - slope) < 0.0005
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // logits.size() == output.size(): the first and last element are both probabilities
invariant softmax_constraint_0
- given input = valid_input()
- then true // logits.size() == output.size()
+ assert softmax_at(1.0, 2.0, valid_input(), 1.0, 0) >= 0.0 and softmax_at(1.0, 2.0, valid_input(), 1.0, 2) >= 0.0
+ // config.temperature >= MIN_TEMP
invariant softmax_constraint_1
- given input = valid_input()
- then true // config.temperature >= MIN_TEMP
+ assert cfg(1.0).temperature >= MIN_TEMP
+ // sum(output) = 1.0 within floating precision
invariant softmax_constraint_2
- given input = valid_input()
- then true // sum(output) ≈ 1.0 (within floating precision)
-
+ assert @abs(softmax_at(1.0, 2.0, valid_input(), 1.0, 0) + softmax_at(1.0, 2.0, valid_input(), 1.0, 1) + softmax_at(1.0, 2.0, valid_input(), 1.0, 2) - 1.0) < 0.0001
diff --git a/apps/website/public/t27/files/specs/ml/activation/tanh_activation.t27 b/apps/website/public/t27/files/specs/ml/activation/tanh_activation.t27
index c96583f4f3..b6044c349c 100644
--- a/apps/website/public/t27/files/specs/ml/activation/tanh_activation.t27
+++ b/apps/website/public/t27/files/specs/ml/activation/tanh_activation.t27
@@ -2,6 +2,11 @@
// t27/specs/
// Derivative (can compute from output) | φ² + 1/φ² = 3 | TRINITY
+// #8980: `math::exp` named a module this spec never imports, and
+// forward_batch allocated with `[]f32{input.len}`. tanh is now computed in the
+// form that cannot overflow: exp(2|x|) reaches inf in f32 at |x| = 45, and
+// inf / inf is NaN.
+
module Tanh;
use base::types;
use math::constants;
@@ -18,55 +23,62 @@ module Tanh;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(x: f32) → f32
+ // tanh(x) = (1 - e^(-2|x|)) / (1 + e^(-2|x|)), with the sign of x
fn forward(x: f32) -> f32 {
- // tanh(x) = (e^x - e^-x) / (e^x + e^-x)
- let exp_x = math::exp(x);
- let exp_neg_x = math::exp(-x);
- return (exp_x - exp_neg_x) / (exp_x + exp_neg_x);
+ if (x < 0.0) {
+ return -forward(-x);
+ }
+ const t = exp(-2.0 * x);
+ return (1.0 - t) / (1.0 + t);
}
- // forward_batch(input: []f32) → []f32
- fn forward_batch(input: []f32) -> []f32 {
- let result = []f32{input.len};
- let mut i = 0;
+ // forward_batch: output[i] = tanh(input[i])
+ fn forward_batch(input: []const f32, output: []f32) -> void {
+ var i : usize = 0;
while (i < input.len) {
- result[i] = forward(input[i]);
+ output[i] = forward(input[i]);
i = i + 1;
}
- return result;
}
- // derivative(x: f32) → f32
+ // tanh'(x) = 1 - tanh(x)^2
fn derivative(x: f32) -> f32 {
- // tanh'(x) = 1 - tanh(x)²
- let tanh_x = forward(x);
+ const tanh_x = forward(x);
return 1.0 - (tanh_x * tanh_x);
}
- // derivative_from_output(tanh_x: f32) → f32
+ // Given y = tanh(x), the derivative is 1 - y^2
fn derivative_from_output(tanh_x: f32) -> f32 {
- // Given tanh(x), the derivative is 1 - tanh(x)²
return 1.0 - (tanh_x * tanh_x);
}
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [0.0, 1.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, &output);
+ return output[index];
+ }
+
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // tanh(0) = 0, tanh(1) = 0.7615942, tanh is odd
test forward_basic_case
given input = 0.0
when result = forward(input)
- then result == 0.0
+ then result == 0.0 and @abs(forward(1.0) - 0.7615942) < 0.000001 and forward(-1.0) == -forward(1.0)
+
+ // exp(2x) is inf in f32 here; the stable form still gives 1, not NaN
+ test forward_saturates_without_overflow
+ given input = 100.0
+ when result = forward(input)
+ then result == 1.0 and forward(-100.0) == -1.0
test forward_batch_basic_case
- given input = []f32{3, [0.0, 1.0, 2.0]}
- when result = forward_batch(input)
- then result.len == 3
- then result[0] == 0.0
- then result[1] > 0.0
- then result[1] < 1.0
- then result[2] > result[1]
+ given first = batch_at(0)
+ when second = batch_at(1)
+ then first == 0.0 and second > 0.0 and second < 1.0 and batch_at(2) > second
test derivative_basic_case
given input = 0.0
@@ -76,6 +88,9 @@ module Tanh;
test derivative_from_output_basic_case
given input = 0.5
when tanh_result = forward(input)
- when result = derivative_from_output(tanh_result)
- then result > 0.0
- then result < 1.0
\ No newline at end of file
+ then derivative_from_output(tanh_result) > 0.0 and derivative_from_output(tanh_result) < 1.0 and derivative_from_output(tanh_result) == derivative(input)
+
+ test derivative_matches_finite_difference
+ given slope = (forward(0.5 + 0.001) - forward(0.5 - 0.001)) / 0.002
+ when result = derivative(0.5)
+ then @abs(result - slope) < 0.001
diff --git a/apps/website/public/t27/files/specs/ml/layers/avgpool2d_layer.t27 b/apps/website/public/t27/files/specs/ml/layers/avgpool2d_layer.t27
index 9429ac4c53..d542f5430a 100644
--- a/apps/website/public/t27/files/specs/ml/layers/avgpool2d_layer.t27
+++ b/apps/website/public/t27/files/specs/ml/layers/avgpool2d_layer.t27
@@ -2,6 +2,12 @@
// t27/specs/
// 2D average pooling formula | φ² + 1/φ² = 3 | TRINITY
+// #8980: the kernel and stride were string constants typed `[]u32`, forward only
+// asserted that its input was non-empty, and the tests asserted `true` or
+// `result != undefined`. Average pooling now runs over one square H x H channel
+// with the default 2 x 2 window and stride 2 and no padding, and each test checks
+// numbers.
+
module Avgpool2d;
use base::types;
use math::constants;
@@ -10,8 +16,8 @@ module Avgpool2d;
// 1. Constants
// ═══════════════════════════════════════════════════════════
- const DEFAULT_KERNEL : []u32 = "[2, 2]";
- const DEFAULT_STRIDE : []u32 = "[2, 2]";
+ const DEFAULT_KERNEL : u32 = 2;
+ const DEFAULT_STRIDE : u32 = 2;
// ═══════════════════════════════════════════════════════════
// 2. Types
@@ -36,36 +42,110 @@ module Avgpool2d;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // calc_output_size(height_in: u32) → void
- fn calc_output_size(height_in: u32) -> void {
- // Calculate output dimensions for 2D average pooling
- // Using default kernel size [2, 2], stride [2, 2], and padding [0, 0]
- let kernel = DEFAULT_KERNEL;
- let stride = DEFAULT_STRIDE;
- let padding = [0u32, 0u32];
-
- // Calculate output height and width
- let height_out = (height_in + 2 * padding[0] - kernel[0]) / stride[0] + 1;
- let width_out = (height_in + 2 * padding[1] - kernel[1]) / stride[1] + 1;
-
- // For now, just validate the calculation
- assert(height_out > 0);
- assert(width_out > 0);
+ // Output side for an input side n: (n - k) / stride + 1, and 0 when no window fits
+ fn calc_output_size(height_in: u32) -> u32 {
+ if (height_in < DEFAULT_KERNEL) {
+ return 0;
+ }
+ return (height_in - DEFAULT_KERNEL) / DEFAULT_STRIDE + 1;
+ }
+
+ // output[oy][ox] = mean of the k x k window at (oy * stride, ox * stride) of a side x side
+ // input; returns the number of outputs written
+ fn forward(input: []const f32, side: u32, output: []f32) -> u32 {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var sum : f32 = 0.0;
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ sum = sum + input[(oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx];
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * n + ox] = sum / area;
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ return n * n;
+ }
+
+ // backward: each window's gradient is shared equally by its k * k inputs
+ fn backward(grad_output: []const f32, side: u32, grad_input: []f32) -> void {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ const idx = (oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx;
+ grad_input[idx] = grad_input[idx] + grad_output[oy * n + ox] / area;
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ }
+
+ // 4 x 4 input 0..15, row-major: windows average to 2.5, 4.5, 10.5, 12.5
+ fn pool4_at(index: usize) -> f32 {
+ var input : [16]f32 = [0.0] ** 16;
+ var i : usize = 0;
+ while (i < 16) {
+ input[i] = @as(f32, @floatFromInt(i));
+ i = i + 1;
+ }
+ var output : [4]f32 = [0.0] ** 4;
+ _ = forward(&input, 4, &output);
+ return output[index];
+ }
+
+ // forward over a 2 x 2 input (1, 2, 3, 4) or a 1 x 1 input (5); returns the outputs written
+ fn pool_count(side: u32) -> u32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const one : [1]f32 = [5.0];
+ var output : [1]f32 = [-1.0];
+ if (side == 2) {
+ return forward(&four, 2, &output);
+ }
+ return forward(&one, 1, &output);
+ }
+
+ fn pool2_value() -> f32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [1]f32 = [-1.0];
+ _ = forward(&four, 2, &output);
+ return output[0];
+ }
+
+ fn backward4_at(index: usize) -> f32 {
+ const grad_output : [4]f32 = [4.0, 8.0, 12.0, 16.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, 4, &grad_input);
+ return grad_input[index];
}
- // forward(input: []f32) → void
- fn forward(input: []f32) -> void {
- // Perform 2D average pooling operation
- // Using default kernel size [2, 2], stride [2, 2], and padding [0, 0]
- let kernel = DEFAULT_KERNEL;
- let stride = DEFAULT_STRIDE;
- let padding = [0u32, 0u32];
-
- // For now, just validate the input
- assert(input.len > 0);
-
- // TODO: Implement actual pooling logic from .tri spec
- // This is a placeholder implementation
+ fn valid_input() -> u32 {
+ return calc_output_size(4);
}
// ═══════════════════════════════════════════════════════════
@@ -73,65 +153,70 @@ module Avgpool2d;
// ═══════════════════════════════════════════════════════════
test calc_output_size_basic_case
- given input = default_input()
- when result = calc_output_size(input)
- then result != undefined
+ given result = calc_output_size(4)
+ then result == 2
test calc_output_size_valid_input
- given input = 10u32
+ given input = 10
when result = calc_output_size(input)
- then true // Function should complete without errors
+ then result == 5
+ // A 1 x 1 input holds no 2 x 2 window
test calc_output_size_small_input
- given input = 1u32
+ given input = 1
when result = calc_output_size(input)
- then true // Function should handle minimal input size
+ then result == 0 and calc_output_size(2) == 1
test calc_output_size_large_input
- given input = 1000u32
+ given input = 1000
when result = calc_output_size(input)
- then true // Function should handle large input size
+ then result == 500 and calc_output_size(1001) == 500
test forward_basic_case
- given input = default_input()
- when result = forward(input)
- then result != undefined
+ given first = pool4_at(0)
+ when last = pool4_at(3)
+ then first == 2.5 and pool4_at(1) == 4.5 and pool4_at(2) == 10.5 and last == 12.5
+ // A 2 x 2 input is one window: the mean of 1, 2, 3, 4
test forward_non_empty_input
- given input = [1.0f32, 2.0f32, 3.0f32, 4.0f32]
- when result = forward(input)
- then true // Function should process valid input array
+ given count = pool_count(2)
+ when value = pool2_value()
+ then count == 1 and value == 2.5
test forward_single_element
- given input = [5.0f32]
- when result = forward(input)
- then true // Function should handle single element input
+ given count = pool_count(1)
+ then count == 0
+
+ // Each of a window's 4 inputs gets a quarter of its gradient
+ test backward_spreads_the_gradient
+ given corner = backward4_at(0)
+ when last = backward4_at(15)
+ then corner == 1.0 and backward4_at(5) == 1.0 and backward4_at(2) == 2.0 and last == 4.0
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // kernel_size[i] >= 1
invariant avgpool2d_constraint_0
- given input = valid_input()
- then true // kernel_size[i] >= 1
+ assert DEFAULT_KERNEL >= 1
+ // stride[i] >= 1
invariant avgpool2d_constraint_1
- given input = valid_input()
- then true // stride[i] >= 1
+ assert DEFAULT_STRIDE >= 1
+ // stride[i] <= kernel_size[i]: windows tile the input without gaps
invariant avgpool2d_constraint_2
- given input = valid_input()
- then true // stride[i] <= kernel_size[i]
+ assert DEFAULT_STRIDE <= DEFAULT_KERNEL
+ // name: avgpool_formula -- (n - k) / s + 1
invariant avgpool2d_constraint_3
- given input = valid_input()
- then true // name: avgpool_formula
+ assert valid_input() == 2
+ // "Zeiler & Fergus (2014) - Visualizing and Understanding Convolutional Networks": pooling halves each side
invariant avgpool2d_constraint_4
- given input = valid_input()
- then true // "Zeiler & Fergus (2014) - Visualizing and Understanding Convolutional Networks"
+ assert calc_output_size(8) == 4
+ // "He et al. (2016) - Deep Residual Learning for Image Recognition": a 7 x 7 map pools to 3 x 3
invariant avgpool2d_constraint_5
- given input = valid_input()
- then true // "He et al. (2016) - Deep Residual Learning for Image Recognition"
-
+ assert calc_output_size(7) == 3
diff --git a/apps/website/public/t27/files/specs/ml/layers/batchnorm_layer.t27 b/apps/website/public/t27/files/specs/ml/layers/batchnorm_layer.t27
index a41052cc7d..e910bfe464 100644
--- a/apps/website/public/t27/files/specs/ml/layers/batchnorm_layer.t27
+++ b/apps/website/public/t27/files/specs/ml/layers/batchnorm_layer.t27
@@ -2,6 +2,12 @@
// t27/specs/
// Prevent division by zero in sqrt | φ² + 1/φ² = 3 | TRINITY
+// #8980: forward copied one element and backward zeroed two gradients
+// (placeholders), and the tests pinned those placeholders: output[0] == input[0],
+// grad_gamma[0] == grad_beta[0] == 0. They now compute batch normalisation:
+// forward/backward in inference form, normalising by the running statistics, and
+// forward_train normalising by the batch's own mean and variance.
+
module Batchnorm;
use base::types;
use math::constants;
@@ -34,79 +40,166 @@ module Batchnorm;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(input: []const f32, gamma: []const f32, beta: []const f32, state: BatchNormState, output: []f32, config: BatchNormConfig) → void
+ // Inference: output[f] = gamma[f] (input[f] - running_mean[f]) / sqrt(running_var[f] + eps) + beta[f]
fn forward(input: []const f32, gamma: []const f32, beta: []const f32, state: BatchNormState, output: []f32, config: BatchNormConfig) -> void {
- // BatchNorm forward implementation
- // Normalize input using running mean and variance, apply gamma and beta scaling
- // Basic implementation - copy input to output
- output[0] = input[0];
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const x_hat = (input[f] - state.running_mean[f]) / sqrt(state.running_var[f] + config.epsilon);
+ output[f] = gamma[f] * x_hat + beta[f];
+ f = f + 1;
+ }
}
- // backward(grad_output: []const f32, input: []const f32, gamma: []const f32, beta: []const f32, mean: []const f32, var: []const f32, grad_gamma: []f32, grad_beta: []f32, grad_input: []f32, config: BatchNormConfig) → void
+ // Inference backward (mean and variance are constants here):
+ // grad_input[f] = grad_output[f] gamma[f] / sqrt(var[f] + eps)
+ // grad_gamma[f] = grad_output[f] x_hat[f], grad_beta[f] = grad_output[f]
fn backward(grad_output: []const f32, input: []const f32, gamma: []const f32, beta: []const f32, mean: []const f32, var: []const f32, grad_gamma: []f32, grad_beta: []f32, grad_input: []f32, config: BatchNormConfig) -> void {
- // BatchNorm backward implementation
- // Compute gradients for gamma, beta, and input
- // Basic implementation - initialize gradients
- grad_gamma[0] = 0.0;
- grad_beta[0] = 0.0;
- grad_input[0] = grad_output[0] * gamma[0] / (sqrt(var[0]) + 1.0e-5);
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const inv_std = 1.0 / sqrt(var[f] + config.epsilon);
+ grad_gamma[f] = grad_output[f] * (input[f] - mean[f]) * inv_std;
+ grad_beta[f] = grad_output[f];
+ grad_input[f] = grad_output[f] * gamma[f] * inv_std;
+ f = f + 1;
+ }
}
- // ═══════════════════════════════════════════════════════════
- // TDD: Tests (from .tri behaviors)
- // ═══════════════════════════════════════════════════════════
+ // Training: input is batch x num_features, row-major. Each feature is normalised by
+ // its own batch mean and (biased) batch variance, then scaled and shifted.
+ fn forward_train(input: []const f32, batch: usize, gamma: []const f32, beta: []const f32, output: []f32, config: BatchNormConfig) -> void {
+ const nf : usize = config.num_features;
+ var f : usize = 0;
+ while (f < nf) {
+ var mean : f32 = 0.0;
+ var b : usize = 0;
+ while (b < batch) {
+ mean = mean + input[b * nf + f];
+ b = b + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(batch));
+ var variance : f32 = 0.0;
+ b = 0;
+ while (b < batch) {
+ const d = input[b * nf + f] - mean;
+ variance = variance + d * d;
+ b = b + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(batch));
+ b = 0;
+ while (b < batch) {
+ output[b * nf + f] = gamma[f] * (input[b * nf + f] - mean) / sqrt(variance + config.epsilon) + beta[f];
+ b = b + 1;
+ }
+ f = f + 1;
+ }
+ }
+
+ fn cfg() -> BatchNormConfig {
+ return BatchNormConfig{ .num_features = 3, .epsilon = DEFAULT_EPSILON, .momentum = DEFAULT_MOMENTUM, .affine = true, .track_running_stats = true };
+ }
+
+ // forward on input (1, 2, 3), gamma g, beta 0.25, running mean 0 / var 1 (or mean 2 / var 4)
+ fn forward_at(g: f32, shifted: bool, index: usize) -> f32 {
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [g, g, g];
+ const beta : [3]f32 = [0.25, 0.25, 0.25];
+ var mean : [3]f32 = [0.0, 0.0, 0.0];
+ var variance : [3]f32 = [1.0, 1.0, 1.0];
+ if (shifted) {
+ mean = [2.0, 2.0, 2.0];
+ variance = [4.0, 4.0, 4.0];
+ }
+ const state = BatchNormState{ .running_mean = &mean, .running_var = &variance };
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&input, &gamma, &beta, state, &output, cfg());
+ return output[index];
+ }
- test "forward_basic_case" {
- var input = []f32{1.0, 2.0, 3.0};
- var gamma = []f32{1.0, 1.0, 1.0};
- var beta = []f32{0.0, 0.0, 0.0};
- var state = BatchNormState{running_mean = []f32{0.0, 0.0, 0.0}, running_var = []f32{1.0, 1.0, 1.0}};
- var output = []f32{0.0, 0.0, 0.0};
- var config = BatchNormConfig{num_features = 3, epsilon = 1.0e-5, momentum = 0.1, affine = true, track_running_stats = true};
-
- forward(input, gamma, beta, state, output, config);
- assert(output[0] == input[0]);
+ // which: 0 = grad_gamma, 1 = grad_beta, 2 = grad_input; grad_output 1, gamma 2, mean 0, var 1
+ fn backward_at(which: u8, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [2.0, 2.0, 2.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ const mean : [3]f32 = [0.0, 0.0, 0.0];
+ const variance : [3]f32 = [1.0, 1.0, 1.0];
+ var grad_gamma : [3]f32 = [9.0] ** 3;
+ var grad_beta : [3]f32 = [9.0] ** 3;
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &gamma, &beta, &mean, &variance, &grad_gamma, &grad_beta, &grad_input, cfg());
+ if (which == 0) {
+ return grad_gamma[index];
+ }
+ if (which == 1) {
+ return grad_beta[index];
+ }
+ return grad_input[index];
}
- test "backward_basic_case" {
- var grad_output = []f32{1.0, 1.0, 1.0};
- var input = []f32{1.0, 2.0, 3.0};
- var gamma = []f32{1.0, 1.0, 1.0};
- var beta = []f32{0.0, 0.0, 0.0};
- var mean = []f32{0.0, 0.0, 0.0};
- var var = []f32{1.0, 1.0, 1.0};
- var grad_gamma = []f32{0.0, 0.0, 0.0};
- var grad_beta = []f32{0.0, 0.0, 0.0};
- var grad_input = []f32{0.0, 0.0, 0.0};
- var config = BatchNormConfig{num_features = 3, epsilon = 1.0e-5, momentum = 0.1, affine = true, track_running_stats = true};
-
- backward(grad_output, input, gamma, beta, mean, var, grad_gamma, grad_beta, grad_input, config);
- assert(grad_gamma[0] == 0.0);
- assert(grad_beta[0] == 0.0);
- assert(grad_input[0] != 0.0);
+ // forward_train on a batch of 4 samples x 3 features; returns output[b][f]
+ fn train_at(b: usize, f: usize) -> f32 {
+ const input : [12]f32 = [1.0, 10.0, -1.0, 2.0, 20.0, -1.0, 3.0, 30.0, 1.0, 4.0, 40.0, 1.0];
+ const gamma : [3]f32 = [1.0, 1.0, 1.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ var output : [12]f32 = [0.0] ** 12;
+ forward_train(&input, 4, &gamma, &beta, &output, cfg());
+ return output[b * 3 + f];
}
+ fn valid_input() -> u32 {
+ return cfg().num_features;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // With mean 0, var 1, gamma 1, beta 0 the layer is the identity up to eps:
+ // 1 / sqrt(1 + 1e-5) = 0.999995 (the old `==` held only for the copy placeholder)
+ test "forward_basic_case"
+ given out0 = forward_at(1.0, false, 0) - 0.25
+ when out2 = forward_at(1.0, false, 2) - 0.25
+ then @abs(out0 - 1.0) < 0.00001 and @abs(out2 - 3.0) < 0.00003
+
+ // mean 2, var 4: (x - 2) / 2 * gamma + beta, so x = 3 with gamma 2 gives 1.25
+ test "forward_uses_running_statistics"
+ given result = forward_at(2.0, true, 2)
+ when centre = forward_at(2.0, true, 1)
+ then @abs(result - 1.25) < 0.00001 and centre == 0.25
+
+ // grad_gamma = g x_hat = (1, 2, 3) / sqrt(1 + eps), grad_beta = g = 1,
+ // grad_input = g gamma / sqrt(1 + eps) = 2 (the old 0, 0 were the placeholder's)
+ test "backward_basic_case"
+ given grad_gamma = backward_at(0, 2)
+ when grad_input = backward_at(2, 0)
+ then @abs(grad_gamma - 3.0) < 0.0001 and backward_at(1, 0) == 1.0 and grad_input != 0.0 and @abs(grad_input - 2.0) < 0.0001
+
+ // Each feature of the batch comes out with mean 0 and variance 1
+ test "forward_train_normalises_each_feature"
+ given mean_f1 = train_at(0, 1) + train_at(1, 1) + train_at(2, 1) + train_at(3, 1)
+ when var_f1 = (train_at(0, 1) * train_at(0, 1) + train_at(1, 1) * train_at(1, 1) + train_at(2, 1) * train_at(2, 1) + train_at(3, 1) * train_at(3, 1)) / 4.0
+ then @abs(mean_f1) < 0.00001 and @abs(var_f1 - 1.0) < 0.0001 and @abs(train_at(0, 0) - train_at(0, 1)) < 0.00001 and @abs(train_at(0, 2) + 1.0) < 0.0001
+
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // num_features > 0
invariant batchnorm_constraint_0
- given input = valid_input()
- then true // num_features > 0
+ assert valid_input() > 0
+ // epsilon > 0
invariant batchnorm_constraint_1
- given input = valid_input()
- then true // epsilon > 0
+ assert DEFAULT_EPSILON > 0.0
+ // 0 <= momentum < 1
invariant batchnorm_constraint_2
- given input = valid_input()
- then true // 0 <= momentum < 1
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+ // gamma.size() == num_features: the fixtures carry 3 of each
invariant batchnorm_constraint_3
- given input = valid_input()
- then true // gamma.size() == num_features
+ assert valid_input() == 3
+ // beta.size() == num_features: beta shifts every feature, so x = mean gives exactly beta
invariant batchnorm_constraint_4
- given input = valid_input()
- then true // beta.size() == num_features
-
+ assert forward_at(2.0, true, 1) == 0.25
diff --git a/apps/website/public/t27/files/specs/ml/layers/dense_layer.t27 b/apps/website/public/t27/files/specs/ml/layers/dense_layer.t27
index be1b0d3274..a75924f73d 100644
--- a/apps/website/public/t27/files/specs/ml/layers/dense_layer.t27
+++ b/apps/website/public/t27/files/specs/ml/layers/dense_layer.t27
@@ -2,6 +2,11 @@
// t27/specs/
// Row-major weight layout for cache efficiency | φ² + 1/φ² = 3 | TRINITY
+// #8980: forward and backward were empty placeholders, the tests asserted
+// `result != undefined`, and the invariants asserted `true` over an undeclared
+// `valid_input`. Dense is now y = act(W x + b) with W row-major
+// (output_size x input_size), and backward gives the three gradients.
+
module Dense;
use base::types;
use math::constants;
@@ -37,67 +42,173 @@ module Dense;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(input: []const f32) → void
- fn forward(input: []const f32) -> void {
- // Dense layer forward pass: output = activation(input * weights + bias)
- // Simple placeholder implementation - just validate input and return
- _ = input; // Prevent unused parameter warning
- return; // Actual executable statement
+ // relu, the one activation the tests exercise besides None
+ fn activate(z: f32, activation: ActivationType) -> f32 {
+ if (activation == ActivationType.ReLU) {
+ if (z > 0.0) {
+ return z;
+ }
+ return 0.0;
+ }
+ return z;
+ }
+
+ // output[o] = act(sum_i weights[o * in + i] * input[i] + bias[o])
+ fn forward(input: []const f32, weights: []const f32, bias: []const f32, config: DenseConfig, output: []f32) -> void {
+ var o : usize = 0;
+ while (o < config.output_size) {
+ var z : f32 = 0.0;
+ if (config.has_bias) {
+ z = bias[o];
+ }
+ var i : usize = 0;
+ while (i < config.input_size) {
+ z = z + weights[o * config.input_size + i] * input[i];
+ i = i + 1;
+ }
+ output[o] = activate(z, config.activation);
+ o = o + 1;
+ }
+ }
+
+ // For a linear layer (activation None), given dL/dy = grad_output:
+ // grad_input[i] = sum_o W[o][i] grad_output[o] (W^T g)
+ // grad_weights[o][i] = grad_output[o] * input[i] (g x^T)
+ // grad_bias[o] = grad_output[o]
+ fn backward(grad_output: []const f32, input: []const f32, weights: []const f32, config: DenseConfig, grad_input: []f32, grad_weights: []f32, grad_bias: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.input_size) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var o : usize = 0;
+ while (o < config.output_size) {
+ grad_bias[o] = grad_output[o];
+ i = 0;
+ while (i < config.input_size) {
+ grad_input[i] = grad_input[i] + weights[o * config.input_size + i] * grad_output[o];
+ grad_weights[o * config.input_size + i] = grad_output[o] * input[i];
+ i = i + 1;
+ }
+ o = o + 1;
+ }
+ }
+
+ // The fixture: 3 inputs, 2 outputs, W = [[1, 2, 3], [-1, 0, 1]], b = [0.5, -0.5]
+ fn cfg(activation: ActivationType) -> DenseConfig {
+ return DenseConfig{ .input_size = 3, .output_size = 2, .has_bias = true, .activation = activation };
+ }
+
+ fn forward_at(x0: f32, x1: f32, x2: f32, activation: ActivationType, index: usize) -> f32 {
+ const input : [3]f32 = [x0, x1, x2];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ const bias : [2]f32 = [0.5, -0.5];
+ var output : [2]f32 = [0.0] ** 2;
+ forward(&input, &weights, &bias, cfg(activation), &output);
+ return output[index];
+ }
+
+ // L = g . y for g = (g0, g1): the loss whose gradients backward computes
+ fn loss_at(x0: f32, x1: f32, x2: f32, g0: f32, g1: f32) -> f32 {
+ return g0 * forward_at(x0, x1, x2, ActivationType.None, 0) + g1 * forward_at(x0, x1, x2, ActivationType.None, 1);
+ }
+
+ // which: 0 = grad_input, 1 = grad_weights, 2 = grad_bias
+ fn backward_at(g0: f32, g1: f32, which: u8, index: usize) -> f32 {
+ const grad_output : [2]f32 = [g0, g1];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ var grad_weights : [6]f32 = [9.0] ** 6;
+ var grad_bias : [2]f32 = [9.0] ** 2;
+ backward(&grad_output, &input, &weights, cfg(ActivationType.None), &grad_input, &grad_weights, &grad_bias);
+ if (which == 0) {
+ return grad_input[index];
+ }
+ if (which == 1) {
+ return grad_weights[index];
+ }
+ return grad_bias[index];
+ }
+
+ // An empty layer (no inputs, no outputs) writes nothing
+ fn empty_forward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ forward(&none, &none, &none, empty_cfg, &sentinel);
+ return sentinel[0] == 7.0;
}
- // backward(grad_output: []const f32) → void
- fn backward(grad_output: []const f32) -> void {
- // Dense layer backward pass: compute gradients through the layer
- // Simple placeholder implementation - just validate gradient and return
- _ = grad_output; // Prevent unused parameter warning
- return; // Actual executable statement
+ fn empty_backward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ backward(&none, &none, &none, empty_cfg, &sentinel, &sentinel, &sentinel);
+ return sentinel[0] == 7.0;
+ }
+
+ fn valid_input() -> f32 {
+ return forward_at(1.0, 2.0, 3.0, ActivationType.None, 0);
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // x = (1, 2, 3): y0 = 1 + 4 + 9 + 0.5 = 14.5, y1 = -1 + 0 + 3 - 0.5 = 1.5
test forward_basic_case
- given input = [1.0f32, 2.0f32, 3.0f32] // 3-element input vector
- when result = forward(input)
- then result != undefined // Function should complete without crashing
+ given y0 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 0)
+ when y1 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 1)
+ then y0 == 14.5 and y1 == 1.5
+
+ // ReLU clips the negative pre-activation: x = (0, 0, -1) gives y = (-2.5, -1.5) -> (0, 0)
+ test forward_relu_clips_negatives
+ given linear = forward_at(0.0, 0.0, -1.0, ActivationType.None, 0)
+ when clipped = forward_at(0.0, 0.0, -1.0, ActivationType.ReLU, 0)
+ then linear == -2.5 and clipped == 0.0 and forward_at(1.0, 2.0, 3.0, ActivationType.ReLU, 0) == 14.5
test forward_empty_input
- given input = []f32 // Empty input
- when result = forward(input)
- then result != undefined // Should handle empty input gracefully
+ given result = empty_forward_untouched()
+ then result
+ // g = (1, 2): grad_input = W^T g = (1 - 2, 2 + 0, 3 + 2) = (-1, 2, 5); grad_bias = g;
+ // grad_weights[1][2] = g1 * x2 = 6
test backward_basic_case
- given grad_output = [0.5f32, -0.3f32, 0.8f32] // 3-element gradient vector
- when result = backward(grad_output)
- then result != undefined // Function should complete without crashing
+ given gi0 = backward_at(1.0, 2.0, 0, 0)
+ when gi2 = backward_at(1.0, 2.0, 0, 2)
+ then gi0 == -1.0 and backward_at(1.0, 2.0, 0, 1) == 2.0 and gi2 == 5.0 and backward_at(1.0, 2.0, 2, 1) == 2.0 and backward_at(1.0, 2.0, 1, 5) == 6.0
+
+ // dL/dx1 by finite difference equals grad_input[1]
+ test backward_matches_finite_difference
+ given slope = (loss_at(1.0, 2.0 + 0.01, 3.0, 1.0, 2.0) - loss_at(1.0, 2.0 - 0.01, 3.0, 1.0, 2.0)) / 0.02
+ when result = backward_at(1.0, 2.0, 0, 1)
+ then @abs(result - slope) < 0.001
test backward_empty_gradient
- given grad_output = []f32 // Empty gradient
- when result = backward(grad_output)
- then result != undefined // Should handle empty gradient gracefully
+ given result = empty_backward_untouched()
+ then result
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // input.size() == config.input_size: the bounds the layer accepts
invariant dense_constraint_0
- given input = valid_input()
- then true // input.size() == config.input_size
+ assert MIN_INPUT_SIZE >= 1 and MAX_INPUT_SIZE >= MIN_INPUT_SIZE
+ // output.size() == config.output_size: each output is one row of W x + b
invariant dense_constraint_1
- given input = valid_input()
- then true // output.size() == config.output_size
+ assert valid_input() == 14.5
+ // weights.size() == input_size * output_size: the fixture holds 3 * 2 = 6 weights
invariant dense_constraint_2
- given input = valid_input()
- then true // weights.size() == input_size * output_size
+ assert cfg(ActivationType.None).input_size * cfg(ActivationType.None).output_size == 6
+ // bias.size() == output_size (if has_bias): the fixture carries a bias of 2
invariant dense_constraint_3
- given input = valid_input()
- then true // bias.size() == output_size (if has_bias)
+ assert cfg(ActivationType.None).has_bias
+ // All inputs must be finite (no NaN/Inf): finite in, finite out
invariant dense_constraint_4
- given input = valid_input()
- then true // All inputs must be finite (no NaN/Inf)
-
+ assert valid_input() == valid_input()
diff --git a/apps/website/public/t27/files/specs/ml/layers/dropout_layer.t27 b/apps/website/public/t27/files/specs/ml/layers/dropout_layer.t27
index 3b09f5e000..4a12f21501 100644
--- a/apps/website/public/t27/files/specs/ml/layers/dropout_layer.t27
+++ b/apps/website/public/t27/files/specs/ml/layers/dropout_layer.t27
@@ -2,6 +2,12 @@
// t27/specs/
// Prevent co-adaptation of neurons | φ² + 1/φ² = 3 | TRINITY
+// #8980: the tests called `allocf32` and `allocbool`, which no spec declares, and
+// asserted `result != undefined`. Inference also scaled by (1 - p) even for
+// inverted dropout, which already scaled by 1 / (1 - p) in training, so the
+// expected activation shrank by (1 - p) between training and inference. Inference
+// is now the identity when scale_during_training is set.
+
module Dropout;
use base::types;
use math::constants;
@@ -44,8 +50,13 @@ module Dropout;
output[i] = 0.0;
}
} else {
- // During inference, always scale output
- output[i] = input[i] * (1.0 - config.p);
+ if (config.scale_during_training) {
+ // Inverted dropout already scaled in training: inference is the identity
+ output[i] = input[i];
+ } else {
+ // Classic dropout: scale by the keep probability at inference
+ output[i] = input[i] * (1.0 - config.p);
+ }
}
i = i + 1;
}
@@ -75,19 +86,48 @@ module Dropout;
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ fn cfg(scaled: bool) -> DropoutConfig {
+ return DropoutConfig{ .p = DEFAULT_DROP_RATE, .inplace = false, .scale_during_training = scaled };
+ }
+
+ // forward on input (1, 2, 3, 4) with mask (keep, drop, keep, drop)
+ fn forward_at(training: bool, scaled: bool, index: usize) -> f32 {
+ const input : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [4]f32 = [9.0] ** 4;
+ var mask : [4]bool = [true, false, true, false];
+ forward(&input, &output, &mask, training, cfg(scaled));
+ return output[index];
+ }
+
+ fn backward_at(scaled: bool, index: usize) -> f32 {
+ const grad_output : [4]f32 = [1.0, 1.0, 1.0, 1.0];
+ const mask : [4]bool = [true, false, true, false];
+ var grad_input : [4]f32 = [9.0] ** 4;
+ backward(&grad_output, &mask, &grad_input, cfg(scaled));
+ return grad_input[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // p = 0.5, inverted: kept units are doubled, dropped ones are 0
test forward_basic_case
- given input = default_input()
- output = allocf32(input.len)
- mask = allocbool(input.len)
- config = DropoutConfig { p = 0.5, inplace = false, scale_during_training = true }
- when result = forward(input, output, mask, true, config)
- then result != undefined
+ given kept = forward_at(true, true, 2)
+ when dropped = forward_at(true, true, 1)
+ then kept == 6.0 and dropped == 0.0 and forward_at(true, true, 0) == 2.0
+
+ // The expected activation is the same in training and inference
+ test forward_inference_keeps_the_expectation
+ given inverted = forward_at(false, true, 2)
+ when classic = forward_at(false, false, 2)
+ then inverted == 3.0 and classic == 1.5 and forward_at(true, false, 2) == 3.0
+ // The gradient flows only through kept units, with the same scale
test backward_basic_case
- given grad_output = default_input()
- mask = allocbool(grad_output.len)
- grad_input = allocf32(grad_output.len)
- config = DropoutConfig { p = 0.5, inplace = false, scale_during_training = true }
- when result = backward(grad_output, mask, grad_input, config)
- then result != undefined
+ given kept = backward_at(true, 0)
+ when dropped = backward_at(true, 3)
+ then kept == 2.0 and dropped == 0.0 and backward_at(false, 2) == 1.0
+ invariant dropout_rate_is_a_probability
+ assert DEFAULT_DROP_RATE >= 0.0 and DEFAULT_DROP_RATE < 1.0
diff --git a/apps/website/public/t27/files/specs/ml/layers/embedding_layer.t27 b/apps/website/public/t27/files/specs/ml/layers/embedding_layer.t27
index 4ed89feeeb..836fe230b7 100644
--- a/apps/website/public/t27/files/specs/ml/layers/embedding_layer.t27
+++ b/apps/website/public/t27/files/specs/ml/layers/embedding_layer.t27
@@ -2,6 +2,12 @@
// t27/specs/
// Embeddings are learned parameters | φ² + 1/φ² = 3 | TRINITY
+// #8980: forward only asserted its arguments and wrote nothing, the tests called
+// undeclared `test_input_forward` / `test_input_init` and asserted
+// `result != undefined`, and the invariants asserted `true`. forward now looks each
+// token's row up in W (vocab_size x d_model, row-major), backward adds each
+// gradient row back into the row it came from, and init fills W deterministically.
+
module Embedding;
use base::types;
use math::constants;
@@ -29,76 +35,131 @@ module Embedding;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(token_ids: []const u32, weights: EmbeddingWeights, embeddings: []f32, config: EmbeddingConfig) → void
+ // embeddings[t * d + j] = W[token_ids[t] * d + j]: one row of W per token
fn forward(token_ids: []const u32, weights: EmbeddingWeights, embeddings: []f32, config: EmbeddingConfig) -> void {
- // Validate input parameters
- assert(config.vocab_size > 0);
- assert(config.d_model > 0);
- assert(weights.W.size() == config.vocab_size * config.d_model);
-
- // Basic implementation: validate all token IDs are in range
- for (i in token_ids) {
- assert(i < config.vocab_size);
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ embeddings[t * d + j] = weights.W[row * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
}
}
- // init(weights: EmbeddingWeights, config: EmbeddingConfig) → void
+ // grad_W[token_ids[t]] += grad_embeddings[t]: a token seen twice collects both rows
+ fn backward(token_ids: []const u32, grad_embeddings: []const f32, grad_W: []f32, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ grad_W[row * d + j] = grad_W[row * d + j] + grad_embeddings[t * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
+ }
+ }
+
+ // Deterministic init in [-0.5, 0.5) / sqrt(d_model): a fixed hash of (row, column), so
+ // two runs start from the same weights
fn init(weights: EmbeddingWeights, config: EmbeddingConfig) -> void {
- // Validate configuration
- assert(config.vocab_size > 0);
- assert(config.d_model > 0);
-
- // Initialize weight matrix with appropriate size
- let expected_size: u32 = config.vocab_size * config.d_model;
- assert(weights.W.size() == expected_size);
-
- // Basic validation - weights array is properly sized
- assert(weights.W.size() > 0);
+ const d : usize = config.d_model;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d)));
+ var k : usize = 0;
+ while (k < config.vocab_size * d) {
+ const h : u32 = @as(u32, @intCast((k * 2654435761) % 1000));
+ weights.W[k] = (@as(f32, @floatFromInt(h)) / 1000.0 - 0.5) * scale;
+ k = k + 1;
+ }
+ }
+
+ // vocab 4, d_model 2, W row v = (10 v, 10 v + 1)
+ fn cfg() -> EmbeddingConfig {
+ return EmbeddingConfig{ .vocab_size = 4, .d_model = 2 };
+ }
+
+ fn lookup_at(a: u32, b: u32, index: usize) -> f32 {
+ var w : [8]f32 = [0.0, 1.0, 10.0, 11.0, 20.0, 21.0, 30.0, 31.0];
+ const ids : [2]u32 = [a, b];
+ var out : [4]f32 = [-1.0] ** 4;
+ forward(&ids, EmbeddingWeights{ .W = &w }, &out, cfg());
+ return out[index];
+ }
+
+ // grad_W[index] after backward of ids (a, b) with gradient rows (1, 2) and (3, 4)
+ fn grad_at(a: u32, b: u32, index: usize) -> f32 {
+ const ids : [2]u32 = [a, b];
+ const grads : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_w : [8]f32 = [0.0] ** 8;
+ backward(&ids, &grads, &grad_w, cfg());
+ return grad_w[index];
+ }
+
+ fn init_at(index: usize) -> f32 {
+ var w : [8]f32 = [9.0] ** 8;
+ init(EmbeddingWeights{ .W = &w }, cfg());
+ return w[index];
+ }
+
+ fn valid_input() -> u32 {
+ return cfg().vocab_size * cfg().d_model;
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // ids (2, 0): rows (20, 21) then (0, 1)
test forward_basic_case
- given input = test_input_forward()
- when result = forward(input)
- then result != undefined
+ given first = lookup_at(2, 0, 0)
+ when last = lookup_at(2, 0, 3)
+ then first == 20.0 and lookup_at(2, 0, 1) == 21.0 and lookup_at(2, 0, 2) == 0.0 and last == 1.0
+ // Every weight is written, inside [-0.5, 0.5) / sqrt(2), and init is reproducible
test init_basic_case
- given input = test_input_init()
- when result = init(input)
- then result != undefined
+ given first = init_at(0)
+ when last = init_at(7)
+ then first != 9.0 and last != 9.0 and @abs(first) <= 0.36 and @abs(last) <= 0.36 and init_at(5) == init_at(5) and init_at(1) != init_at(2)
+
+ // A token seen twice collects both gradient rows; an unseen row stays 0
+ test backward_accumulates_repeated_tokens
+ given row1 = grad_at(1, 1, 2)
+ when unseen = grad_at(1, 1, 0)
+ then row1 == 4.0 and grad_at(1, 1, 3) == 6.0 and unseen == 0.0 and grad_at(1, 3, 6) == 3.0
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // vocab_size > 0
invariant embedding_constraint_0
- given input = valid_input()
- then true // vocab_size > 0
+ assert cfg().vocab_size > 0
+ // d_model > 0
invariant embedding_constraint_1
- given input = valid_input()
- then true // d_model > 0
+ assert cfg().d_model > 0 and DEFAULT_D_MODEL > 0
+ // W.size() == vocab_size * d_model
invariant embedding_constraint_2
- given input = valid_input()
- then true // W.size() == vocab_size * d_model
+ assert valid_input() == 8
+ // token_ids must be in [0, vocab_size): the last id reads the last row
invariant embedding_constraint_3
- given input = valid_input()
- then true // token_ids must be in [0, vocab_size)
+ assert lookup_at(3, 3, 1) == 31.0
+ // "Embedding dimension affects model capacity": each token gets d_model numbers
invariant embedding_constraint_4
- given input = valid_input()
- then true // "Embedding dimension affects model capacity"
+ assert lookup_at(1, 2, 1) == 11.0 and lookup_at(1, 2, 2) == 20.0
+ // "Larger vocab requires larger d_model": the init scale shrinks as 1 / sqrt(d_model)
invariant embedding_constraint_5
- given input = valid_input()
- then true // "Larger vocab requires larger d_model"
+ assert 1.0 / sqrt(@as(f32, 64.0)) < 1.0 / sqrt(@as(f32, 2.0))
+ // "Pre-trained embeddings transfer well": forward is a pure lookup, so loaded rows come out unchanged
invariant embedding_constraint_6
- given input = valid_input()
- then true // "Pre-trained embeddings transfer well"
-
+ assert lookup_at(0, 1, 0) == 0.0 and lookup_at(0, 1, 2) == 10.0
diff --git a/apps/website/public/t27/files/specs/ml/layers/maxpool2d_layer.t27 b/apps/website/public/t27/files/specs/ml/layers/maxpool2d_layer.t27
index f73b9ddab9..7b972a41a5 100644
--- a/apps/website/public/t27/files/specs/ml/layers/maxpool2d_layer.t27
+++ b/apps/website/public/t27/files/specs/ml/layers/maxpool2d_layer.t27
@@ -2,6 +2,13 @@
// t27/specs/
// Reduce spatial dimensions | φ² + 1/φ² = 3 | TRINITY
+// #8980: forward and backward compared two values and threw them away
+// (placeholders), and every test asserted `result != undefined`. Max pooling now
+// runs over one H x W channel (row-major) with a k x k window and a stride. It
+// records each window's argmax, and backward sends each gradient back to that
+// element only. Padding and dilation are carried in the config but not read: the
+// layer assumes padding 0 and dilation 1.
+
module Maxpool2d;
use base::types;
use math::constants;
@@ -22,81 +29,141 @@ module Maxpool2d;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(input: []const f32) → void
- fn forward(input: []const f32) -> void {
- // Max pooling forward pass: find maximum values in each window
- // Simple implementation that processes input data
- const input_size : u32 = @truncate(input.len);
-
- // Basic max pooling logic - process first few elements as example
- if (input_size > 1) {
- const val1 : f32 = input[0];
- const val2 : f32 = input[1];
-
- // Simple max operation
- if (val1 > val2) {
- _ = val1; // Store max value (simplified)
- } else {
- _ = val2; // Store max value (simplified)
- }
- }
-
- if (input_size > 3) {
- const val3 : f32 = input[2];
- const val4 : f32 = input[3];
-
- // Simple max operation
- if (val3 > val4) {
- _ = val3; // Store max value (simplified)
- } else {
- _ = val4; // Store max value (simplified)
+ // Output side for an input side n: (n - k) / stride + 1
+ fn out_size(n: u32, config: MaxPool2DConfig) -> u32 {
+ return (n - config.kernel_size) / config.stride + 1;
+ }
+
+ // output[oy][ox] = max of the k x k window at (oy * stride, ox * stride);
+ // indices[oy][ox] = the flat input index that held it
+ fn forward(input: []const f32, height: u32, width: u32, config: MaxPool2DConfig, output: []f32, indices: []u32) -> void {
+ const oh = out_size(height, config);
+ const ow = out_size(width, config);
+ var oy : u32 = 0;
+ while (oy < oh) {
+ var ox : u32 = 0;
+ while (ox < ow) {
+ var best_index : u32 = (oy * config.stride) * width + ox * config.stride;
+ var ky : u32 = 0;
+ while (ky < config.kernel_size) {
+ var kx : u32 = 0;
+ while (kx < config.kernel_size) {
+ const idx : u32 = (oy * config.stride + ky) * width + ox * config.stride + kx;
+ if (input[idx] > input[best_index]) {
+ best_index = idx;
+ }
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * ow + ox] = input[best_index];
+ indices[oy * ow + ox] = best_index;
+ ox = ox + 1;
}
+ oy = oy + 1;
}
}
- // backward(grad_output: []const f32) → void
- fn backward(grad_output: []const f32) -> void {
- // Max pooling backward pass: propagate gradients to input
- // Simple implementation that processes gradient data
- const grad_size : u32 = @truncate(grad_output.len);
-
- // Basic gradient propagation logic - process first few elements as example
- if (grad_size > 0) {
- const grad1 : f32 = grad_output[0];
- _ = grad1; // Process gradient value (simplified)
+ // grad_input[indices[o]] += grad_output[o]; every other input gets 0
+ fn backward(grad_output: []const f32, indices: []const u32, grad_input: []f32) -> void {
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
}
-
- if (grad_size > 1) {
- const grad2 : f32 = grad_output[1];
- _ = grad2; // Process gradient value (simplified)
+ var o : usize = 0;
+ while (o < grad_output.len) {
+ grad_input[indices[o]] = grad_input[indices[o]] + grad_output[o];
+ o = o + 1;
}
}
+ fn cfg(kernel: u32, stride: u32) -> MaxPool2DConfig {
+ return MaxPool2DConfig{ .kernel_size = kernel, .stride = stride, .padding = 0, .dilation = 1, .return_indices = true };
+ }
+
+ // The output side on the 4 x 4 fixture for a k x k window and a stride.
+ // out_size takes a MaxPool2DConfig struct and does not lower to WASM; this
+ // takes two integers, so a host reads the side from the compiled spec.
+ fn out_side(kernel: u32, stride: u32) -> u32 {
+ return out_size(4, cfg(kernel, stride));
+ }
+
+ // The 4 x 4 fixture
+ // 1 2 5 6
+ // 3 4 8 7
+ // 0 -1 -3 -2
+ // -4 9 -6 -5
+ fn pool_at(kernel: u32, stride: u32, which: u8, index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [9]f32 = [0.0] ** 9;
+ var indices : [9]u32 = [0] ** 9;
+ forward(&input, 4, 4, cfg(kernel, stride), &output, &indices);
+ if (which == 0) {
+ return output[index];
+ }
+ return @as(f32, @floatFromInt(indices[index]));
+ }
+
+ // 2 x 2, stride 2, gradients (1, 2, 3, 4) for the four windows; grad_input[index]
+ fn backward_at(index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [4]f32 = [0.0] ** 4;
+ var indices : [4]u32 = [0] ** 4;
+ forward(&input, 4, 4, cfg(2, 2), &output, &indices);
+ const grad_output : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, &indices, &grad_input);
+ return grad_input[index];
+ }
+
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // 2 x 2, stride 2: windows' maxima are 4, 8, 9, -2
test forward_basic_case
- given input = default_input()
- when result = forward(input)
- then result != undefined
- then input.len > 0 // Input should not be empty
-
- test backward_basic_case
- given grad_output = [1.0, 2.0, 3.0, 4.0] // Sample gradient output
- when result = backward(grad_output)
- then result != undefined
- then grad_output.len > 0 // Gradient input should not be empty
+ given first = pool_at(2, 2, 0, 0)
+ when last = pool_at(2, 2, 0, 3)
+ then first == 4.0 and pool_at(2, 2, 0, 1) == 8.0 and pool_at(2, 2, 0, 2) == 9.0 and last == -2.0
+ // The argmax indices: 4 at 5, 8 at 6, 9 at 13, -2 at 11
test forward_max_finding
- given input = [1.0, 2.0, 3.0, 4.0] // 2x2 input
- when result = forward(input)
- then result != undefined
- // Should find max values in each window
+ given first = pool_at(2, 2, 1, 0)
+ when third = pool_at(2, 2, 1, 2)
+ then first == 5.0 and pool_at(2, 2, 1, 1) == 6.0 and third == 13.0 and pool_at(2, 2, 1, 3) == 11.0
+ // 3 x 3, stride 1 gives a 2 x 2 output whose windows overlap
+ test forward_overlapping_windows
+ given first = pool_at(3, 1, 0, 0)
+ when last = pool_at(3, 1, 0, 3)
+ then first == 8.0 and pool_at(3, 1, 0, 1) == 8.0 and pool_at(3, 1, 0, 2) == 9.0 and last == 9.0
+
+ test backward_basic_case
+ given at_max = backward_at(5)
+ when elsewhere = backward_at(0)
+ then at_max == 1.0 and elsewhere == 0.0
+
+ // Each window's gradient lands on its argmax only: the sum is conserved
test backward_gradient_propagation
- given grad_output = [0.5, 1.0] // Sample gradients
- when result = backward(grad_output)
- then result != undefined
- // Should properly propagate gradients
+ given total = backward_at(5) + backward_at(6) + backward_at(13) + backward_at(11)
+ when others = backward_at(0) + backward_at(1) + backward_at(4) + backward_at(15)
+ then total == 10.0 and others == 0.0 and backward_at(13) == 3.0
+
+ // The four window/stride pairs the nn-maxpool widget offers, on the 4 x 4 fixture:
+ // 2 x 2 stride 2 -> 2, 3 x 3 stride 1 -> 2, 2 x 2 stride 1 -> 3, 4 x 4 stride 1 -> 1
+ test out_side_of_each_widget_config
+ given halved = out_side(2, 2)
+ when whole = out_side(4, 1)
+ then halved == 2 and out_side(3, 1) == 2 and out_side(2, 1) == 3 and whole == 1
+
+ // Edges: a 1 x 1 window at stride 1 keeps the side; a stride that does not
+ // divide (4 - k) rounds down (3 x 3 at stride 2 fits one window, 1 x 1 at
+ // stride 3 fits two)
+ test out_side_identity_and_floor
+ given same = out_side(1, 1)
+ when floored = out_side(3, 2)
+ then same == 4 and floored == 1 and out_side(1, 3) == 2
+ invariant maxpool_output_size_formula
+ assert out_size(4, cfg(2, 2)) == 2 and out_size(4, cfg(3, 1)) == 2 and out_size(5, cfg(2, 2)) == 2
diff --git a/apps/website/public/t27/files/specs/ml/loss/binary_crossentropy_loss.t27 b/apps/website/public/t27/files/specs/ml/loss/binary_crossentropy_loss.t27
index 5f398bb75a..063ee794ce 100644
--- a/apps/website/public/t27/files/specs/ml/loss/binary_crossentropy_loss.t27
+++ b/apps/website/public/t27/files/specs/ml/loss/binary_crossentropy_loss.t27
@@ -2,6 +2,12 @@
// t27/specs/
// Gradient w.r.t predictions | φ² + 1/φ² = 3 | TRINITY
+// #8980: never compiled (`new f32[n]`, a float loop index, `predictions.size`,
+// an empty enum), and it approximated sigmoid and log with polynomials that
+// were 0.3 off at x = 1. The probability gradient also had the wrong sign on its
+// second term: d/dp of -(t log p + (1-t) log(1-p)) is -t/p + (1-t)/(1-p). It now
+// uses exp and log, writes gradients into a caller's buffer, and reduces by mean.
+
module BinaryCe;
use base::types;
use math::constants;
@@ -22,168 +28,114 @@ module BinaryCe;
reduction : Reduction,
};
- pub const Reduction = struct {
- enum : ,
+ pub const Reduction = enum(u8) {
+ Mean,
+ Sum,
};
// ═══════════════════════════════════════════════════════════
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // sigmoid(x: f32) → f32
fn sigmoid(x: f32) -> f32 {
- // Simple sigmoid approximation using rational function
- if (x > 6.0) return 1.0;
- if (x < -6.0) return 0.0;
-
- var abs_x = x;
- if (abs_x < 0.0) abs_x = -abs_x;
-
- // Polynomial approximation for sigmoid
- var p = 0.3178722 + abs_x * (0.3655024 + abs_x * (0.1811160 + abs_x * (0.0443368 + abs_x * 0.0086892)));
- var p_inv = 1.0 / p;
-
- if (x > 0.0) {
- return 1.0 - p_inv;
- } else {
- return p_inv;
+ if (x >= 0.0) {
+ return 1.0 / (1.0 + exp(-x));
}
+ const e = exp(x);
+ return e / (1.0 + e);
}
- // Helper function for log approximation
- fn log_approx(x: f32) -> f32 {
- // Simple log approximation for x in (0, 1]
- if (x <= 0.0) return 10.0; // Large number for invalid input
- if (x >= 1.0) return 0.0;
-
- // Polynomial approximation for log(x) around x=1
- var t = 1.0 - x;
- if (t > 0.5) {
- // For x < 0.5, use log(x) = -log(1/x)
- return -log_approx(1.0 / x);
+ // p clipped into [eps, 1 - eps] so log stays finite
+ fn clip(p: f32, eps: f32) -> f32 {
+ if (p < eps) {
+ return eps;
+ }
+ if (p > 1.0 - eps) {
+ return 1.0 - eps;
}
-
- // Taylor series: log(1-t) = -t - t^2/2 - t^3/3 - ...
- var result = -t;
- var t2 = t * t;
- var t3 = t2 * t;
- var t4 = t3 * t;
-
- result = result - t2 * 0.5 - t3 * 0.333333 - t4 * 0.25;
- return result;
+ return p;
}
- // forward(predictions: []f32, targets: []f32, config: BinaryCEConfig) → f32
- fn forward(predictions: []f32, targets: []f32, config: BinaryCEConfig) -> f32 {
- var loss: f32 = 0.0;
- var i: f32 = 0.0;
- while (i < predictions.size) {
- var pred: f32 = predictions[f32(i)];
- var target: f32 = targets[f32(i)];
- i = i + 1.0;
-
- // Apply sigmoid if needed
- if (config.from_logits) {
- pred = sigmoid(pred);
- }
-
- // Clip predictions to avoid numerical issues
- pred = pred + config.epsilon;
- if (pred > 1.0 - config.epsilon) {
- pred = 1.0 - config.epsilon;
- }
-
- // Simplified binary cross-entropy approximation
- if (target > 0.5) {
- // Target is 1, loss is -log(pred)
- loss = loss + -log_approx(pred);
- } else {
- // Target is 0, loss is -log(1-pred)
- loss = loss + -log_approx(1.0 - pred);
- }
+ // -(t log p + (1 - t) log(1 - p)) for one element
+ fn element_loss(pred: f32, target: f32, config: BinaryCEConfig) -> f32 {
+ var p = pred;
+ if (config.from_logits) {
+ p = sigmoid(pred);
}
-
- // Apply reduction (mean)
- return loss / f32(predictions.size);
+ p = clip(p, config.epsilon);
+ return -(target * log(p) + (1.0 - target) * log(1.0 - p));
}
- // forward_batch(predictions: [][]f32, targets: []f32, config: BinaryCEConfig) → f32
- fn forward_batch(predictions: [][]f32, targets: []f32, config: BinaryCEConfig) -> f32 {
- var total_loss: f32 = 0.0;
- var total_samples: f32 = 0.0;
-
- var batch_idx: f32 = 0.0;
- while (batch_idx < predictions.size) {
- var batch_preds = predictions[f32(batch_idx)];
- var i: f32 = 0.0;
- while (i < batch_preds.size) {
- var pred: f32 = batch_preds[f32(i)];
- var target: f32 = targets[f32(batch_idx) * batch_preds.size + f32(i)];
- i = i + 1.0;
-
- // Apply sigmoid if needed
- if (config.from_logits) {
- pred = sigmoid(pred);
- }
-
- // Clip predictions to avoid numerical issues
- pred = pred + config.epsilon;
- if (pred > 1.0 - config.epsilon) {
- pred = 1.0 - config.epsilon;
- }
-
- // Simplified binary cross-entropy approximation
- if (target > 0.5) {
- // Target is 1, loss is -log(pred)
- total_loss = total_loss + -log_approx(pred);
- } else {
- // Target is 0, loss is -log(1-pred)
- total_loss = total_loss + -log_approx(1.0 - pred);
- }
- }
-
- total_samples = total_samples + f32(batch_preds.size);
+ // mean (or sum) of element_loss
+ fn forward(predictions: []const f32, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ loss = loss + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ if (config.reduction == Reduction.Sum) {
+ return loss;
}
-
- // Apply reduction (mean across all samples)
- return total_loss / total_samples;
+ return loss / @as(f32, @floatFromInt(predictions.len));
}
- // gradient(predictions: []f32, targets: []f32, config: BinaryCEConfig) → []f32
- fn gradient(predictions: []f32, targets: []f32, config: BinaryCEConfig) -> []f32 {
- var grads: []f32 = new f32[predictions.size];
- var i: f32 = 0.0;
- while (i < predictions.size) {
- var pred: f32 = predictions[f32(i)];
- var target: f32 = targets[f32(i)];
- i = i + 1.0;
- var pred: f32 = predictions[i];
- var target: f32 = targets[i];
-
- // Apply sigmoid if needed
- if (config.from_logits) {
- pred = sigmoid(pred);
- }
-
- // Clip predictions to avoid division by zero
- pred = pred + config.epsilon;
- if (pred > 1.0 - config.epsilon) {
- pred = 1.0 - config.epsilon;
- }
-
- // Compute gradient
+ // A batch of rows of width `width`, flattened row-major: the mean over every element
+ fn forward_batch(predictions: []const f32, width: usize, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ total = total + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ _ = width;
+ return total / @as(f32, @floatFromInt(predictions.len));
+ }
+
+ // Per-element gradient of the (unreduced) loss:
+ // from logits: sigmoid(z) - t
+ // from probabilities: -t / p + (1 - t) / (1 - p)
+ fn gradient(predictions: []const f32, targets: []const f32, config: BinaryCEConfig, grads: []f32) -> void {
+ var i : usize = 0;
+ while (i < predictions.len) {
+ const t = targets[i];
if (config.from_logits) {
- // For logits, gradient is (pred - target)
- grads[i] = pred - target;
+ grads[i] = sigmoid(predictions[i]) - t;
} else {
- // For probabilities, gradient is -(target/pred + (1-target)/(1-pred))
- var inv_pred = 1.0 / pred;
- var inv_1_minus_pred = 1.0 / (1.0 - pred);
- grads[i] = -(target * inv_pred + (1.0 - target) * inv_1_minus_pred);
+ const p = clip(predictions[i], config.epsilon);
+ grads[i] = -t / p + (1.0 - t) / (1.0 - p);
}
+ i = i + 1;
}
-
- return grads;
+ }
+
+ fn cfg(from_logits: bool) -> BinaryCEConfig {
+ return BinaryCEConfig{ .from_logits = from_logits, .epsilon = DEFAULT_EPSILON, .reduction = Reduction.Mean };
+ }
+
+ // The fixture: predictions (a, b, c) against targets (1, 0, 1)
+ fn loss_of(a: f32, b: f32, c: f32, from_logits: bool) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ return forward(&predictions, &targets, cfg(from_logits));
+ }
+
+ fn grad_of(a: f32, b: f32, c: f32, from_logits: bool, index: usize) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ var grads : [3]f32 = [9.0] ** 3;
+ gradient(&predictions, &targets, cfg(from_logits), &grads);
+ return grads[index];
+ }
+
+ fn batch_loss() -> f32 {
+ const predictions : [4]f32 = [0.8, 0.3, 0.9, 0.1];
+ const targets : [4]f32 = [1.0, 0.0, 1.0, 0.0];
+ return forward_batch(&predictions, 2, &targets, cfg(false));
+ }
+
+ fn valid_input() -> f32 {
+ return loss_of(0.8, 0.3, 0.9, false);
}
// ═══════════════════════════════════════════════════════════
@@ -193,78 +145,70 @@ module BinaryCe;
test sigmoid_basic_case
given input = 0.0
when result = sigmoid(input)
- then abs(result - 0.5) < 1e-6
+ then @abs(result - 0.5) < 1e-6
test sigmoid_positive_input
given input = 1.0
when result = sigmoid(input)
- then result > 0.5 and result < 1.0
+ then result > 0.5 and result < 1.0 and @abs(result - 0.7310586) < 0.000001
test sigmoid_negative_input
given input = -1.0
when result = sigmoid(input)
- then result > 0.0 and result < 0.5
+ then result > 0.0 and result < 0.5 and @abs(result + sigmoid(1.0) - 1.0) < 0.000001
+ // -(ln 0.8 + ln 0.7 + ln 0.9) / 3 = 0.2283930
test forward_basic_case
- given predictions = [0.8, 0.3, 0.9]
- given targets = [1.0, 0.0, 1.0]
- given config = BinaryCEConfig{from_logits: false, epsilon: 1e-7, reduction: undefined}
- when result = forward(predictions, targets, config)
- then result > 0.0
+ given result = loss_of(0.8, 0.3, 0.9, false)
+ then result > 0.0 and @abs(result - 0.2283930) < 0.000001
+ // From logits (2, -1, 3) it is the same loss as from their sigmoids
test forward_from_logits_case
- given predictions = [2.0, -1.0, 3.0]
- given targets = [1.0, 0.0, 1.0]
- given config = BinaryCEConfig{from_logits: true, epsilon: 1e-7, reduction: undefined}
- when result = forward(predictions, targets, config)
- then result > 0.0
+ given result = loss_of(2.0, -1.0, 3.0, true)
+ when probs = loss_of(sigmoid(2.0), sigmoid(-1.0), sigmoid(3.0), false)
+ then result > 0.0 and @abs(result - probs) < 0.000001
+ // (-ln 0.8 - ln 0.7 - ln 0.9 - ln 0.9) / 4 = 0.1976349
test forward_batch_basic_case
- given predictions = [[0.8, 0.3], [0.9, 0.1]]
- given targets = [1.0, 0.0, 1.0, 0.0]
- given config = BinaryCEConfig{from_logits: false, epsilon: 1e-7, reduction: undefined}
- when result = forward_batch(predictions, targets, config)
- then result > 0.0
+ given result = batch_loss()
+ then result > 0.0 and @abs(result - 0.1976349) < 0.000001
+ // t = 1 pulls p up (negative gradient), t = 0 pushes it down (positive)
test gradient_basic_case
- given predictions = [0.8, 0.3, 0.9]
- given targets = [1.0, 0.0, 1.0]
- given config = BinaryCEConfig{from_logits: false, epsilon: 1e-7, reduction: undefined}
- when result = gradient(predictions, targets, config)
- then result.length == 3 and result[0] < 0.0 and result[1] > 0.0 and result[2] < 0.0
+ given g0 = grad_of(0.8, 0.3, 0.9, false, 0)
+ when g1 = grad_of(0.8, 0.3, 0.9, false, 1)
+ then g0 < 0.0 and g1 > 0.0 and grad_of(0.8, 0.3, 0.9, false, 2) < 0.0 and @abs(g0 + 1.25) < 0.000001 and @abs(g1 - 1.0 / 0.7) < 0.000001
+ // From logits: sigmoid(z) - t, and it matches a finite difference of the loss
test gradient_from_logits_case
- given predictions = [2.0, -1.0, 3.0]
- given targets = [1.0, 0.0, 1.0]
- given config = BinaryCEConfig{from_logits: true, epsilon: 1e-7, reduction: undefined}
- when result = gradient(predictions, targets, config)
- then result.length == 3
+ given g1 = grad_of(2.0, -1.0, 3.0, true, 1)
+ when slope = 3.0 * (loss_of(2.0, -1.0 + 0.001, 3.0, true) - loss_of(2.0, -1.0 - 0.001, 3.0, true)) / 0.002
+ then @abs(g1 - sigmoid(-1.0)) < 0.000001 and @abs(g1 - slope) < 0.001
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // targets[i] in {0, 1}: a perfect prediction costs (almost) nothing
invariant binary_ce_constraint_0
- given input = valid_input()
- then true // targets[i] ∈ {0, 1}
+ assert loss_of(1.0, 0.0, 1.0, false) < 0.000001
+ // 0 < epsilon << 1
invariant binary_ce_constraint_1
- given input = valid_input()
- then true // 0 < epsilon << 1
+ assert DEFAULT_EPSILON > 0.0 and DEFAULT_EPSILON < 0.001
+ // name: binary_ce -- positive for an imperfect prediction
invariant binary_ce_constraint_2
- given input = valid_input()
- then true // name: binary_ce
+ assert valid_input() > 0.0
+ // name: gradient -- zero where the logit already gives the target
invariant binary_ce_constraint_3
- given input = valid_input()
- then true // name: gradient
+ assert @abs(grad_of(30.0, -30.0, 30.0, true, 0)) < 0.000001
+ // "Bishop (2006) - Pattern Recognition and Machine Learning": the loss falls as p nears t
invariant binary_ce_constraint_4
- given input = valid_input()
- then true // "Bishop (2006) - Pattern Recognition and Machine Learning"
+ assert loss_of(0.9, 0.1, 0.9, false) < loss_of(0.6, 0.4, 0.6, false)
+ // "Goodfellow et al. (2016) - Deep Learning": clipping keeps a certain wrong answer finite
invariant binary_ce_constraint_5
- given input = valid_input()
- then true // "Goodfellow et al. (2016) - Deep Learning"
-
+ assert loss_of(0.0, 1.0, 0.0, false) < 20.0
diff --git a/apps/website/public/t27/files/specs/ml/optimizer/adam.t27 b/apps/website/public/t27/files/specs/ml/optimizer/adam.t27
index 1492ebd727..df8bd6334d 100644
--- a/apps/website/public/t27/files/specs/ml/optimizer/adam.t27
+++ b/apps/website/public/t27/files/specs/ml/optimizer/adam.t27
@@ -2,6 +2,12 @@
// t27/specs/
// Adam adapts learning rate per parameter | φ² + 1/φ² = 3 | TRINITY
+// #8980: step only checked its gradients with an undeclared `math::is_finite`
+// and updated nothing; the tests asserted `result != undefined`. step is now
+// Adam (Kingma and Ba, 2015) with bias correction. It refuses (returns false,
+// changes nothing) when a gradient is NaN or infinite, which is what the old body
+// validated.
+
module Adam;
use base::types;
use math::constants;
@@ -30,57 +36,122 @@ module Adam;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // step(grads: []const f32) → void
- fn step(grads: []const f32) -> void {
- // Adam optimizer step implementation
- // Update parameters using Adam algorithm:
- // m_t = β1 * m_{t-1} + (1 - β1) * g_t
- // v_t = β2 * v_{t-1} + (1 - β2) * g_t^2
- // m̂_t = m_t / (1 - β1^t)
- // v̂_t = v_t / (1 - β2^t)
- // θ_t = θ_{t-1} - learning_rate * m̂_t / (√v̂_t + ε)
-
- // For this basic implementation, we'll validate input gradients
- // and demonstrate the Adam algorithm structure
- if (grads.len == 0) {
- return; // No gradients to process
+ // b^t for a whole t >= 0
+ fn powi(b: f32, t: u32) -> f32 {
+ var r : f32 = 1.0;
+ var n : u32 = 0;
+ while (n < t) {
+ r = r * b;
+ n = n + 1;
}
-
- // In a full implementation, this would:
- // 1. Initialize first and second moment estimates if not already initialized
- // 2. Update parameters with bias-corrected estimates
- // 3. Apply weight decay if configured
- // 4. Track step count for bias correction
-
- // For now, validate that gradients are reasonable values
- var i: usize = 0;
+ return r;
+ }
+
+ // NaN is the one value not equal to itself; inf - inf is NaN
+ fn is_finite(x: f32) -> bool {
+ return x == x and (x - x) == (x - x);
+ }
+
+ // One Adam step number t (1-based) over params, with moment buffers m and v:
+ // g = grad + wd p
+ // m = b1 m + (1 - b1) g v = b2 v + (1 - b2) g^2
+ // m_hat = m / (1 - b1^t) v_hat = v / (1 - b2^t)
+ // p = p - lr m_hat / (sqrt(v_hat) + eps)
+ fn step(params: []f32, grads: []const f32, m: []f32, v: []f32, t: u32, config: AdamConfig) -> bool {
+ var i : usize = 0;
while (i < grads.len) {
- // Ensure gradients are finite numbers
- if (math::is_finite(grads[i]) == false) {
- return; // Invalid gradient value
+ if (!is_finite(grads[i])) {
+ return false;
}
- i += 1;
+ i = i + 1;
+ }
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ i = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ m[i] = config.beta1 * m[i] + (1.0 - config.beta1) * g;
+ v[i] = config.beta2 * v[i] + (1.0 - config.beta2) * g * g;
+ const m_hat = m[i] / c1;
+ const v_hat = v[i] / c2;
+ params[i] = params[i] - config.learning_rate * m_hat / (sqrt(v_hat) + config.epsilon);
+ i = i + 1;
+ }
+ return true;
+ }
+
+ fn cfg(lr: f32) -> AdamConfig {
+ return AdamConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.0 };
+ }
+
+ // which: 0 = param, 1 = m, 2 = v after one step on params (1, 2, 3) with grads (g0, g1, g2)
+ fn first_step(g0: f32, g1: f32, g2: f32, which: u8, index: usize) -> f32 {
+ var params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [g0, g1, g2];
+ var m : [3]f32 = [0.0] ** 3;
+ var v : [3]f32 = [0.0] ** 3;
+ _ = step(¶ms, &grads, &m, &v, 1, cfg(0.001));
+ if (which == 1) {
+ return m[index];
+ }
+ if (which == 2) {
+ return v[index];
+ }
+ return params[index];
+ }
+
+ // a / b at run time, where IEEE gives NaN for 0 / 0 and inf for 1 / 0
+ fn ratio(a: f32, b: f32) -> f32 {
+ return a / b;
+ }
+
+ fn accepts(g: f32) -> bool {
+ var params : [1]f32 = [1.0];
+ const grads : [1]f32 = [g];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ return step(¶ms, &grads, &m, &v, 1, cfg(0.001)) and params[0] != 1.0;
+ }
+
+ // Minimise (x - 3)^2 from 0
+ fn descend(steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ var t : u32 = 1;
+ while (t <= steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ _ = step(¶ms, &grads, &m, &v, t, cfg(0.1));
+ t = t + 1;
}
+ return params[0];
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // Bias correction makes the first step lr * sign(g), whatever |g| is
test step_basic_case
- given input = default_input()
- when result = step(input)
- then result != undefined
+ given p0 = first_step(0.1, -5.0, 0.0, 0, 0)
+ when p1 = first_step(0.1, -5.0, 0.0, 0, 1)
+ then @abs(p0 - 0.999) < 0.000001 and @abs(p1 - 2.001) < 0.000001 and first_step(0.1, -5.0, 0.0, 0, 2) == 3.0
+ // m = 0.1 g and v = 0.001 g^2 after the first step
test step_with_gradients
- given input = [0.1, 0.2, 0.3]
- when result = step(input)
- then result != undefined
- // Verify function handles gradient input without crashing
+ given m2 = first_step(0.1, 0.2, 0.3, 1, 2)
+ when v2 = first_step(0.1, 0.2, 0.3, 2, 2)
+ then @abs(m2 - 0.03) < 0.0000001 and @abs(v2 - 0.00009) < 0.0000001
+ // Finite gradients are applied; NaN and inf are refused
test step_validates_gradients
- given input = [1.5, -2.3, 0.0, 4.7]
- when result = step(input)
- then result != undefined
- // Verify function processes valid gradients correctly
+ given ok = accepts(1.5) and accepts(-2.3) and accepts(4.7)
+ when refused = !accepts(ratio(0.0, 0.0)) and !accepts(ratio(1.0, 0.0))
+ then ok and refused
+
+ test step_descends_a_quadratic
+ given result = descend(500)
+ then @abs(result - 3.0) < 0.01
+ invariant adam_betas_are_below_one
+ assert DEFAULT_BETA1 < 1.0 and DEFAULT_BETA2 < 1.0 and DEFAULT_BETA1 < DEFAULT_BETA2 and DEFAULT_EPSILON > 0.0
diff --git a/apps/website/public/t27/files/specs/ml/optimizer/lamb.t27 b/apps/website/public/t27/files/specs/ml/optimizer/lamb.t27
index 804fbc372d..5d9e8dfb42 100644
--- a/apps/website/public/t27/files/specs/ml/optimizer/lamb.t27
+++ b/apps/website/public/t27/files/specs/ml/optimizer/lamb.t27
@@ -2,9 +2,18 @@
// t27/specs/
// LAMB parameter update with trust ratio clipping | φ² + 1/φ² = 3 | TRINITY
+// #8980: built with `append` on `[]` and `range` / `len` / `pow`, so it never
+// compiled. The math was also not LAMB: v_hat used beta1's bias correction, the
+// "trust ratio" was ||g|| / ||v_hat|| and was computed, then never applied to the
+// step, and m and v were never written back. It is now You et al. (2020):
+// r = m_hat / (sqrt(v_hat) + eps) + wd * w, trust = min(||w||, clip) / ||r||,
+// w = w - lr * trust * r,
+// per layer, over buffers the caller owns.
+
module Lamb;
use base::types;
use math::constants;
+ use ml::optimizer::adam;
// ═══════════════════════════════════════════════════════════
// 1. Constants
@@ -39,115 +48,138 @@ module Lamb;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // init_state(num_params: u32) → LAMBState
- fn init_state(num_params: u32) -> LAMBState {
- let m = [];
- let v = [];
- // Initialize moment arrays with zeros
- for i in range(num_params) {
- append(m, 0.0);
- append(v, 0.0);
+ // m and v are the first num_params entries of the given buffers, zeroed; step 0
+ fn init_state(m: []f32, v: []f32, num_params: u32) -> LAMBState {
+ var i : usize = 0;
+ while (i < num_params) {
+ m[i] = 0.0;
+ v[i] = 0.0;
+ i = i + 1;
}
- let step = 0u32;
- return LAMBState { m: m, v: v, step: step };
+ return LAMBState{ .m = m[0..num_params], .v = v[0..num_params], .step = 0 };
}
- // compute_layer_update(params: []f32, grads: []f32, state: LAMBState, config: LAMBConfig) → []f32
- fn compute_layer_update(params: []f32, grads: []f32, state: LAMBState, config: LAMBConfig) -> []f32 {
- let num_params = len(params);
- let m = state.m;
- let v = state.v;
- let step = state.step;
-
- // Update biased first moment estimate
- let m_next = [];
- for i in range(num_params) {
- let m_i = config.beta1 * m[i] + (1.0 - config.beta1) * grads[i];
- append(m_next, m_i);
+ // powi (b^t for a whole t) comes from specs/ml/optimizer/adam.t27 through `use`
+
+ // Writes the moments for step (state.step + 1) into state.m / state.v and the update to
+ // ADD to params into `update`; returns update[0..n]
+ fn compute_layer_update(params: []const f32, grads: []const f32, state: LAMBState, config: LAMBConfig, update: []f32) -> []f32 {
+ const n = params.len;
+ const t = state.step + 1;
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ var w_norm : f32 = 0.0;
+ var r_norm : f32 = 0.0;
+ var i : usize = 0;
+ while (i < n) {
+ state.m[i] = config.beta1 * state.m[i] + (1.0 - config.beta1) * grads[i];
+ state.v[i] = config.beta2 * state.v[i] + (1.0 - config.beta2) * grads[i] * grads[i];
+ const r = (state.m[i] / c1) / (sqrt(state.v[i] / c2) + config.epsilon) + config.weight_decay * params[i];
+ update[i] = r;
+ w_norm = w_norm + params[i] * params[i];
+ r_norm = r_norm + r * r;
+ i = i + 1;
}
-
- // Update biased second raw moment estimate
- let v_next = [];
- for i in range(num_params) {
- let v_i = config.beta2 * v[i] + (1.0 - config.beta2) * grads[i] * grads[i];
- append(v_next, v_i);
+ w_norm = sqrt(w_norm);
+ r_norm = sqrt(r_norm);
+ var trust : f32 = 1.0;
+ if (w_norm > 0.0 and r_norm > 0.0) {
+ trust = @min(w_norm, config.clip_threshold) / r_norm;
}
-
- // Compute bias-corrected first and second moment estimates
- let m_hat = [];
- let v_hat = [];
- let correction = 1.0 - pow(config.beta1, f32(step + 1));
- for i in range(num_params) {
- append(m_hat, m_next[i] / correction);
- append(v_hat, v_next[i] / correction);
+ i = 0;
+ while (i < n) {
+ update[i] = -config.learning_rate * trust * update[i];
+ i = i + 1;
}
-
- // Compute trust ratio for adaptive clipping
- let trust_ratio = 0.0;
- let v_hat_sum = 0.0;
- let grad_norm = 0.0;
- for i in range(num_params) {
- v_hat_sum += v_hat[i];
- grad_norm += grads[i] * grads[i];
+ return update[0..n];
+ }
+
+ // One LAMB step over several layers packed back to back in `params` (layer k has
+ // sizes[k] entries), each with its own state; params change in place, each step + 1.
+ fn forward(params: []f32, grads: []const f32, sizes: []const usize, states: []LAMBState, config: LAMBConfig, scratch: []f32) -> void {
+ var start : usize = 0;
+ var k : usize = 0;
+ while (k < sizes.len) {
+ const end = start + sizes[k];
+ const upd = compute_layer_update(params[start..end], grads[start..end], states[k], config, scratch);
+ var j : usize = 0;
+ while (j < sizes[k]) {
+ params[start + j] = params[start + j] + upd[j];
+ j = j + 1;
+ }
+ states[k].step = states[k].step + 1;
+ start = end;
+ k = k + 1;
}
- let v_hat_norm = sqrt(v_hat_sum);
- let grad_norm_val = sqrt(grad_norm);
- if v_hat_norm > 0.0 and grad_norm_val > 0.0 {
- trust_ratio = grad_norm_val / v_hat_norm;
+ }
+
+ fn cfg(lr: f32) -> LAMBConfig {
+ return LAMBConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.01, .clip_threshold = DEFAULT_CLIP_THRESHOLD };
+ }
+
+ // which: 0 = m.len, 1 = v.len, 2 = step, 3 = m[0], 4 = v[99]
+ fn init_probe(which: u8) -> f32 {
+ var m : [100]f32 = [9.0] ** 100;
+ var v : [100]f32 = [9.0] ** 100;
+ const st = init_state(&m, &v, 100);
+ if (which == 0) {
+ return @as(f32, @floatFromInt(st.m.len));
}
-
- // Apply adaptive clipping
- let clipped_grads = [];
- for i in range(num_params) {
- let clipped_grad = grads[i];
- if trust_ratio > config.clip_threshold {
- clipped_grad = clipped_grad * (config.clip_threshold / trust_ratio);
- }
- append(clipped_grads, clipped_grad);
+ if (which == 1) {
+ return @as(f32, @floatFromInt(st.v.len));
}
-
- // Apply weight decay and compute update
- let update = [];
- for i in range(num_params) {
- let update_i = (-config.learning_rate) * (m_hat[i] / (sqrt(v_hat[i]) + config.epsilon) + config.weight_decay * params[i]);
- append(update, update_i);
+ if (which == 2) {
+ return @as(f32, @floatFromInt(st.step));
}
-
- return update;
+ if (which == 3) {
+ return st.m[0];
+ }
+ return st.v[99];
}
- // forward(layers: [][]f32, grads: [][]f32, states: []LAMBState, config: LAMBConfig) → [][]f32
- fn forward(layers: [][]f32, grads: [][]f32, states: []LAMBState, config: LAMBConfig) -> [][]f32 {
- let num_layers = len(layers);
- let updated_layers = [];
-
- for i in range(num_layers) {
- let layer = layers[i];
- let layer_grads = grads[i];
- let state = states[i];
-
- // Compute layer updates using LAMB algorithm
- let updates = compute_layer_update(layer, layer_grads, state, config);
-
- // Apply updates to get updated layer
- let updated_layer = [];
- for j in range(len(layer)) {
- let updated_param = layer[j] + updates[j];
- append(updated_layer, updated_param);
- }
-
- // Update state for next iteration
- let updated_state = LAMBState {
- m: state.m, // These should be updated in compute_layer_update, but for now keep as is
- v: state.v,
- step: state.step + 1u32,
- };
-
- // Store updated state (in a real implementation, we'd update the states array)
- append(updated_layers, updated_layer);
+ // update[index] of the first step on params (1, 2, 3), grads (0.1, 0.2, 0.3); index 3 = its length
+ fn first_update(index: usize) -> f32 {
+ const params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [0.1, 0.2, 0.3];
+ var m : [3]f32 = undefined;
+ var v : [3]f32 = undefined;
+ var update : [3]f32 = [9.0] ** 3;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 3), cfg(0.001), &update);
+ if (index == 3) {
+ return @as(f32, @floatFromInt(u.len));
}
-
- return updated_layers;
+ return u[index];
+ }
+
+ // The layer-wise property: scaling a layer's weights by 10 scales its step by 10
+ fn step_norm(scale: f32) -> f32 {
+ const params : [2]f32 = [1.0 * scale, 2.0 * scale];
+ const grads : [2]f32 = [0.1, 0.2];
+ var m : [2]f32 = undefined;
+ var v : [2]f32 = undefined;
+ var update : [2]f32 = undefined;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 2), cfg(0.001), &update);
+ return sqrt(u[0] * u[0] + u[1] * u[1]);
+ }
+
+ // Two layers (1, 2) and (3, 4), one step; which: 0..3 = new params, 4 = step of layer 1
+ fn two_layers(which: usize) -> f32 {
+ var params : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const grads : [4]f32 = [0.1, 0.2, 0.3, 0.4];
+ const sizes : [2]usize = [2, 2];
+ var m : [4]f32 = undefined;
+ var v : [4]f32 = undefined;
+ var states : [2]LAMBState = [init_state(m[0..2], v[0..2], 2), init_state(m[2..4], v[2..4], 2)];
+ var scratch : [2]f32 = undefined;
+ forward(¶ms, &grads, &sizes, &states, cfg(0.001), &scratch);
+ if (which == 4) {
+ return @as(f32, @floatFromInt(states[1].step));
+ }
+ return params[which];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_BETA1;
}
// ═══════════════════════════════════════════════════════════
@@ -155,89 +187,60 @@ module Lamb;
// ═══════════════════════════════════════════════════════════
test init_state_basic_case
- given num_params = 100u32
- when result = init_state(num_params)
- then len(result.m) == num_params
- and len(result.v) == num_params
- and result.step == 0u32
- and result.m[0] == 0.0
- and result.v[0] == 0.0
+ given num_params = 100.0
+ when m_len = init_probe(0)
+ then m_len == num_params and init_probe(1) == num_params and init_probe(2) == 0.0 and init_probe(3) == 0.0 and init_probe(4) == 0.0
+ // Every update is negative (descent) and the step's norm is lr * min(||w||, 10) = 0.001 ||w||
test compute_layer_update_basic_case
- given params = [1.0, 2.0, 3.0]
- given grads = [0.1, 0.2, 0.3]
- given state = init_state(3u32)
- given config = LAMBConfig {
- learning_rate: 0.001,
- beta1: 0.9,
- beta2: 0.999,
- epsilon: 1e-6,
- weight_decay: 0.01,
- clip_threshold: 10.0
- }
- when result = compute_layer_update(params, grads, state, config)
- then len(result) == len(params)
- and result[0] != params[0] // Update should change the parameter
- and result[1] != params[1]
- and result[2] != params[2]
- and result[0] < 0.0 // Updates should be negative (gradient descent)
- and result[1] < 0.0
- and result[2] < 0.0
+ given u0 = first_update(0)
+ when n = first_update(3)
+ then n == 3.0 and u0 < 0.0 and first_update(1) < 0.0 and first_update(2) < 0.0 and @abs(sqrt(u0 * u0 + first_update(1) * first_update(1) + first_update(2) * first_update(2)) - 0.001 * sqrt(@as(f32, 14.0))) < 0.0000001
+ // Each layer moves, and each layer's state advances
test forward_basic_case
- given layers = [[1.0, 2.0], [3.0, 4.0]]
- given grads = [[0.1, 0.2], [0.3, 0.4]]
- given states = [init_state(2u32), init_state(2u32)]
- given config = LAMBConfig {
- learning_rate: 0.001,
- beta1: 0.9,
- beta2: 0.999,
- epsilon: 1e-6,
- weight_decay: 0.01,
- clip_threshold: 10.0
- }
- when result = forward(layers, grads, states, config)
- then len(result) == len(layers)
- and len(result[0]) == len(layers[0])
- and len(result[1]) == len(layers[1])
- and result[0][0] != layers[0][0] // First layer should be updated
- and result[0][1] != layers[0][1]
- and result[1][0] != layers[1][0] // Second layer should be updated
- and result[1][1] != layers[1][1]
+ given p0 = two_layers(0)
+ when p3 = two_layers(3)
+ then p0 != 1.0 and two_layers(1) != 2.0 and two_layers(2) != 3.0 and p3 != 4.0 and p0 < 1.0 and two_layers(4) == 1.0
+
+ // LAMB's point: the step is proportional to the layer's weight norm (up to the clip)
+ test compute_layer_update_scales_with_the_layer
+ given small = step_norm(1.0)
+ when large = step_norm(3.0)
+ then @abs(large / small - 3.0) < 0.001
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // 0 < beta1 < 1
invariant lamb_constraint_0
- given input = valid_input()
- then true // 0 < beta1 < 1
+ assert valid_input() > 0.0 and valid_input() < 1.0
+ // 0 < beta2 < 1
invariant lamb_constraint_1
- given input = valid_input()
- then true // 0 < beta2 < 1
+ assert DEFAULT_BETA2 > 0.0 and DEFAULT_BETA2 < 1.0
+ // epsilon > 0
invariant lamb_constraint_2
- given input = valid_input()
- then true // epsilon > 0
+ assert DEFAULT_EPSILON > 0.0
+ // clip_threshold > 0
invariant lamb_constraint_3
- given input = valid_input()
- then true // clip_threshold > 0
+ assert DEFAULT_CLIP_THRESHOLD > 0.0
+ // name: trust_ratio -- capped by clip_threshold: a layer of norm 50 steps as one of norm 10
invariant lamb_constraint_4
- given input = valid_input()
- then true // name: trust_ratio
+ assert DEFAULT_CLIP_THRESHOLD == 10.0
+ // name: lamb_update -- bias correction makes the first m_hat equal g
invariant lamb_constraint_5
- given input = valid_input()
- then true // name: lamb_update
+ assert powi(DEFAULT_BETA1, 1) == DEFAULT_BETA1 and DEFAULT_LR > 0.0
+ // "You et al. (2020) - Large Batch Optimization for Deep Learning: Training BERT in 76 minutes"
invariant lamb_constraint_6
- given input = valid_input()
- then true // "You et al. (2020) - Large Batch Optimization for Deep Learning: Training BERT in 76 minutes"
+ assert powi(0.5, 3) == 0.125
+ // "Used in BERT, T5, and other large-scale Transformer training"
invariant lamb_constraint_7
- given input = valid_input()
- then true // "Used in BERT, T5, and other large-scale Transformer training"
-
+ assert DEFAULT_BETA2 > DEFAULT_BETA1
diff --git a/apps/website/public/t27/files/specs/ml/optimizer/sgd.t27 b/apps/website/public/t27/files/specs/ml/optimizer/sgd.t27
index c2616d1101..af7c998fa1 100644
--- a/apps/website/public/t27/files/specs/ml/optimizer/sgd.t27
+++ b/apps/website/public/t27/files/specs/ml/optimizer/sgd.t27
@@ -2,6 +2,12 @@
// t27/specs/
// Nesterov accelerated gradient (NAG) | φ² + 1/φ² = 3 | TRINITY
+// #8980: init called `new f32[n]`; update read undeclared locals; dampening and
+// nesterov were ignored; and weight decay was ADDED to the parameter
+// (`- v + wd * p`), which grows the weights it should shrink. update is now
+// PyTorch's SGD: g = grad + wd p; v = mu v + (1 - dampening) g;
+// step = g + mu v (Nesterov) or v; p = p - lr step.
+
module Sgd;
use base::types;
use math::constants;
@@ -33,74 +39,126 @@ module Sgd;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // init(state: SGDState, param_count: u32) → void
+ // init: the first param_count velocities start at zero (the caller owns the buffer)
fn init(state: SGDState, param_count: u32) -> void {
- // Initialize velocity array with zeros
- state.velocity = new f32[param_count];
+ var i : usize = 0;
+ while (i < param_count) {
+ state.velocity[i] = 0.0;
+ i = i + 1;
+ }
}
- // update(params: []f32, grads: []const f32, state: SGDState, config: SGDConfig) -> void
fn update(params: []f32, grads: []const f32, state: SGDState, config: SGDConfig) -> void {
- // Nesterov accelerated gradient descent update
- i = 0;
- while (i < params.size()) {
- velocity = state.velocity[i];
- grad = grads[i];
-
- // Classic momentum update with Nesterov acceleration
- velocity_new = config.momentum * velocity + config.learning_rate * grad;
- state.velocity[i] = velocity_new;
-
- // Update parameter with velocity and weight decay
- params[i] = params[i] - velocity_new + config.weight_decay * params[i];
+ var i : usize = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ var step = g;
+ if (config.momentum > 0.0) {
+ state.velocity[i] = config.momentum * state.velocity[i] + (1.0 - config.dampening) * g;
+ if (config.nesterov) {
+ step = g + config.momentum * state.velocity[i];
+ } else {
+ step = state.velocity[i];
+ }
+ }
+ params[i] = params[i] - config.learning_rate * step;
i = i + 1;
}
}
+ fn cfg(lr: f32, momentum: f32, wd: f32, nesterov: bool) -> SGDConfig {
+ return SGDConfig{ .learning_rate = lr, .momentum = momentum, .weight_decay = wd, .dampening = 0.0, .nesterov = nesterov };
+ }
+
+ // Minimise f(x) = (x - 3)^2 (gradient 2 (x - 3)) from x = 0 for `steps` steps
+ fn descend(lr: f32, momentum: f32, nesterov: bool, steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var velocity : [1]f32 = [5.0];
+ const state = SGDState{ .velocity = &velocity };
+ init(state, 1);
+ var n : u32 = 0;
+ while (n < steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ update(¶ms, &grads, state, cfg(lr, momentum, 0.0, nesterov));
+ n = n + 1;
+ }
+ return params[0];
+ }
+
+ fn init_zeroes(index: usize) -> f32 {
+ var velocity : [3]f32 = [7.0, 7.0, 7.0];
+ init(SGDState{ .velocity = &velocity }, 3);
+ return velocity[index];
+ }
+
+ // One step on params (1, 2), grads (0.1, 0.2), momentum 0.9, weight decay 0.001
+ fn one_step(index: usize) -> f32 {
+ var params : [2]f32 = [1.0, 2.0];
+ const grads : [2]f32 = [0.1, 0.2];
+ var velocity : [2]f32 = [0.0, 0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.01, 0.9, 0.001, false));
+ return params[index];
+ }
+
+ // A step with zero gradient and weight decay shrinks the weight toward 0
+ fn decay_step() -> f32 {
+ var params : [1]f32 = [2.0];
+ const grads : [1]f32 = [0.0];
+ var velocity : [1]f32 = [0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.1, 0.0, 0.5, false));
+ return params[0];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_LR;
+ }
+
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
- test "init_basic_case" {
- // init takes the state by value, so the velocity it allocates is not observable
- // here; this runs it on a fresh state rather than asserting what it cannot show.
- var empty: [0]f32 = [];
- var state = SGDState{ .velocity = &empty };
- init(state, 3);
- }
+ test "init_basic_case"
+ given first = init_zeroes(0)
+ when last = init_zeroes(2)
+ then first == 0.0 and last == 0.0
- test "update_basic_case" {
- var params: [2]f32 = [1.0, 2.0];
- var grads: [2]f32 = [0.1, 0.2];
- var velocity: [2]f32 = [0.0, 0.0];
- var state = SGDState{ .velocity = &velocity };
- var config = SGDConfig{ .learning_rate = 0.01, .momentum = 0.9, .weight_decay = 0.001, .dampening = 0.0, .nesterov = false };
- update(¶ms, &grads, state, config);
- assert(params[0] != 1.0); // updated
- assert(params[1] != 2.0); // updated
- }
+ // v = g + wd p (momentum starts at 0): p0 = 1 - 0.01 (0.1 + 0.001) = 0.99899
+ test "update_basic_case"
+ given p0 = one_step(0)
+ when p1 = one_step(1)
+ then p0 != 1.0 and p1 != 2.0 and @abs(p0 - 0.99899) < 0.000001 and @abs(p1 - (2.0 - 0.01 * 0.202)) < 0.000001
+
+ // Weight decay pulls the weight toward 0: 2 - 0.1 * 0.5 * 2 = 1.9
+ test "update_weight_decay_shrinks"
+ given result = decay_step()
+ then @abs(result - 1.9) < 0.000001
+
+ // Plain, momentum and Nesterov SGD all reach the minimum at 3
+ test "update_descends_a_quadratic"
+ given plain = descend(0.1, 0.0, false, 100)
+ when heavy = descend(0.05, 0.9, false, 300)
+ then @abs(plain - 3.0) < 0.0001 and @abs(heavy - 3.0) < 0.0001 and @abs(descend(0.05, 0.9, true, 300) - 3.0) < 0.0001
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // learning_rate > 0
invariant sgd_constraint_0
- given input = valid_input()
- then true // learning_rate > 0
+ assert valid_input() > 0.0
+ // 0 <= momentum < 1
invariant sgd_constraint_1
- given input = valid_input()
- then true // 0 <= momentum < 1
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+ // weight_decay >= 0: a non-negative decay never grows a weight with zero gradient
invariant sgd_constraint_2
- given input = valid_input()
- then true // weight_decay >= 0
+ assert decay_step() < 2.0
+ // params.size() == grads.size(): every parameter moves
invariant sgd_constraint_3
- given input = valid_input()
- then true // params.size() == grads.size()
+ assert one_step(0) != 1.0 and one_step(1) != 2.0
+ // If momentum > 0: state.velocity.size() == params.size() -- init zeroes all of it
invariant sgd_constraint_4
- given input = valid_input()
- then true // If momentum > 0: state.velocity.size() == params.size()
-
+ assert init_zeroes(1) == 0.0
diff --git a/apps/website/public/t27/files/specs/ml/pathway/mlp.t27 b/apps/website/public/t27/files/specs/ml/pathway/mlp.t27
index 787801ba76..df03c810ed 100644
--- a/apps/website/public/t27/files/specs/ml/pathway/mlp.t27
+++ b/apps/website/public/t27/files/specs/ml/pathway/mlp.t27
@@ -2,6 +2,13 @@
// t27/specs/
// ReLU activation for hidden layers | φ² + 1/φ² = 3 | TRINITY
+// #8980: forward computed the hidden layer and wrote it into `output` (the
+// second layer never ran); init called an undeclared `alloc_f32`; the tests
+// called undeclared `create_test_*` helpers; and the invariants asserted `true`.
+// The MLP is now two layers, h = relu(W1 x + b1) and y = W2 h + b2, over buffers
+// the caller owns. train_step adds backpropagation with an SGD update, and the
+// tests train the network on XOR, which no single linear layer can fit.
+
module Mlp;
use base::types;
use math::constants;
@@ -35,129 +42,283 @@ module Mlp;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(input: []const f32, state: MLPState, output: []f32, config: MLPConfig) → void
- fn forward(input: []const f32, state: MLPState, output: []f32, config: MLPConfig) -> void {
- // Forward pass implementation for MLP
- // ReLU activation for hidden layers
- // Process input through the network layers
-
- // First layer: input to hidden with ReLU activation
- var i : usize = 0;
- while (i < config.hidden_size) : (i += 1) {
- var sum : f32 = 0.0;
- var j : usize = 0;
- while (j < config.input_size) : (j += 1) {
- sum += input[j] * state.w1[i * config.input_size + j];
+ // One dense layer: out[o] = sum_i w[o * n_in + i] in[i] + b[o], relu when asked
+ fn layer(input: []const f32, w: []const f32, b: []const f32, n_in: u32, n_out: u32, relu: bool, out: []f32) -> void {
+ var o : usize = 0;
+ while (o < n_out) {
+ var sum : f32 = b[o];
+ var i : usize = 0;
+ while (i < n_in) {
+ sum = sum + w[o * n_in + i] * input[i];
+ i = i + 1;
}
- sum += state.b1[i];
- // ReLU activation
- if (sum > 0.0) {
- output[i] = sum;
- } else {
- output[i] = 0.0;
+ if (relu and sum < 0.0) {
+ sum = 0.0;
}
+ out[o] = sum;
+ o = o + 1;
+ }
+ }
+
+ // h = relu(W1 x + b1) into `hidden`, y = W2 h + b2 into `output`
+ fn forward(input: []const f32, state: MLPState, hidden: []f32, output: []f32, config: MLPConfig) -> void {
+ layer(input, state.w1, state.b1, config.input_size, config.hidden_size, true, hidden);
+ layer(hidden, state.w2, state.b2, config.hidden_size, config.output_size, false, output);
+ }
+
+ // He-scaled deterministic init: w in [-1, 1) sqrt(2 / fan_in) from a fixed hash of the
+ // index (salt tells the layers apart); every bias 0
+ fn init_weights(w: []f32, fan_in: u32, salt: usize) -> void {
+ const scale : f32 = sqrt(2.0 / @as(f32, @floatFromInt(fan_in)));
+ var k : usize = 0;
+ while (k < w.len) {
+ const h : u32 = @as(u32, @intCast(((k + salt) * 2654435761) % 1000));
+ w[k] = (@as(f32, @floatFromInt(h)) / 500.0 - 1.0) * scale;
+ k = k + 1;
+ }
+ }
+
+ fn zero(v: []f32) -> void {
+ var k : usize = 0;
+ while (k < v.len) {
+ v[k] = 0.0;
+ k = k + 1;
}
}
- // init(state: MLPState, config: MLPConfig) → void
fn init(state: MLPState, config: MLPConfig) -> void {
- // Initialize MLP state with default values
- // Allocate weight and bias matrices with proper dimensions
- state.w1 = alloc_f32(config.input_size * config.hidden_size);
- state.b1 = alloc_f32(config.hidden_size);
- state.w2 = alloc_f32(config.hidden_size * config.output_size);
- state.b2 = alloc_f32(config.output_size);
-
- // Initialize weights with small random values
+ init_weights(state.w1, config.input_size, 1);
+ zero(state.b1);
+ init_weights(state.w2, config.hidden_size, 7);
+ zero(state.b2);
+ }
+
+ // One SGD step on the loss 0.5 * sum_o (y_o - t_o)^2; returns the loss before the step.
+ // d2_o = y_o - t_o (output error)
+ // d1_i = [z_i > 0] sum_o W2[o][i] d2_o (backpropagated through relu)
+ // W2 -= lr d2 h^T, b2 -= lr d2; W1 -= lr d1 x^T, b1 -= lr d1
+ fn train_step(input: []const f32, target: []const f32, state: MLPState, hidden: []f32, delta: []f32, output: []f32, lr: f32, config: MLPConfig) -> f32 {
+ forward(input, state, hidden, output, config);
+ var loss : f32 = 0.0;
+ var o : usize = 0;
+ while (o < config.output_size) {
+ const e = output[o] - target[o];
+ loss = loss + 0.5 * e * e;
+ o = o + 1;
+ }
var i : usize = 0;
- while (i < config.input_size * config.hidden_size) : (i += 1) {
- state.w1[i] = 0.01; // Small fixed values for now
+ while (i < config.hidden_size) {
+ var back : f32 = 0.0;
+ if (hidden[i] > 0.0) {
+ o = 0;
+ while (o < config.output_size) {
+ back = back + state.w2[o * config.hidden_size + i] * (output[o] - target[o]);
+ o = o + 1;
+ }
+ }
+ delta[i] = back;
+ i = i + 1;
}
-
- // Initialize biases to zero
- i = 0;
- while (i < config.hidden_size) : (i += 1) {
- state.b1[i] = 0.0;
+ o = 0;
+ while (o < config.output_size) {
+ const d2 = output[o] - target[o];
+ i = 0;
+ while (i < config.hidden_size) {
+ state.w2[o * config.hidden_size + i] = state.w2[o * config.hidden_size + i] - lr * d2 * hidden[i];
+ i = i + 1;
+ }
+ state.b2[o] = state.b2[o] - lr * d2;
+ o = o + 1;
}
-
- // Initialize second layer weights with small random values
i = 0;
- while (i < config.hidden_size * config.output_size) : (i += 1) {
- state.w2[i] = 0.01; // Small fixed values for now
+ while (i < config.hidden_size) {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ state.w1[i * config.input_size + j] = state.w1[i * config.input_size + j] - lr * delta[i] * input[j];
+ j = j + 1;
+ }
+ state.b1[i] = state.b1[i] - lr * delta[i];
+ i = i + 1;
}
-
- // Initialize second layer biases to zero
- i = 0;
- while (i < config.output_size) : (i += 1) {
- state.b2[i] = 0.0;
+ return loss;
+ }
+
+ fn mnist() -> MLPConfig {
+ return MLPConfig{ .input_size = MNIST_INPUT_SIZE, .hidden_size = DEFAULT_HIDDEN_SIZE, .output_size = MNIST_OUTPUT_SIZE };
+ }
+
+ // which: 0 = output[index], 1 = min over every output, 2 = number of outputs written,
+ // on the MNIST shape with input pixel k = (k % 7) / 7
+ fn mnist_forward(which: u8, index: usize) -> f32 {
+ var w1 : [784 * 128]f32 = undefined;
+ var b1 : [128]f32 = undefined;
+ var w2 : [128 * 10]f32 = undefined;
+ var b2 : [10]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, mnist());
+ var input : [784]f32 = undefined;
+ var k : usize = 0;
+ while (k < 784) {
+ input[k] = @as(f32, @floatFromInt(k % 7)) / 7.0;
+ k = k + 1;
+ }
+ var hidden : [128]f32 = undefined;
+ var output : [11]f32 = [-99.0] ** 11;
+ forward(&input, state, &hidden, &output, mnist());
+ if (which == 0) {
+ return output[index];
+ }
+ var min_h : f32 = hidden[0];
+ var written : f32 = 0.0;
+ k = 0;
+ while (k < 128) {
+ min_h = @min(min_h, hidden[k]);
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 11) {
+ if (output[k] != -99.0) {
+ written = written + 1.0;
+ }
+ k = k + 1;
+ }
+ if (which == 1) {
+ return min_h;
+ }
+ return written;
+ }
+
+ // which: 0 = w1.len written by init, 1 = b1[0], 2 = b2[0], 3 = largest |w1|
+ fn mnist_init(which: u8) -> f32 {
+ var w1 : [784 * 128]f32 = [9.0] ** (784 * 128);
+ var b1 : [128]f32 = [9.0] ** 128;
+ var w2 : [128 * 10]f32 = [9.0] ** (128 * 10);
+ var b2 : [10]f32 = [9.0] ** 10;
+ init(MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, mnist());
+ var written : f32 = 0.0;
+ var largest : f32 = 0.0;
+ var k : usize = 0;
+ while (k < w1.len) {
+ if (w1[k] != 9.0) {
+ written = written + 1.0;
+ }
+ largest = @max(largest, @abs(w1[k]));
+ k = k + 1;
+ }
+ if (which == 0) {
+ return written;
}
+ if (which == 1) {
+ return b1[0];
+ }
+ if (which == 2) {
+ return b2[0];
+ }
+ return largest;
+ }
+
+ // Train a 2-8-1 network on XOR for `epochs` passes over the four cases.
+ // which: 0 = total loss of the first epoch, 1 = total loss of the last epoch,
+ // 2 + c = the trained network's output for case c
+ fn xor(epochs: u32, which: usize) -> f32 {
+ const cfg = MLPConfig{ .input_size = 2, .hidden_size = 8, .output_size = 1 };
+ var w1 : [16]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [8]f32 = undefined;
+ var b2 : [1]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, cfg);
+ const xs : [8]f32 = [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0];
+ const ts : [4]f32 = [0.0, 1.0, 1.0, 0.0];
+ var hidden : [8]f32 = undefined;
+ var delta : [8]f32 = undefined;
+ var output : [1]f32 = undefined;
+ var first : f32 = 0.0;
+ var last : f32 = 0.0;
+ var e : u32 = 0;
+ while (e < epochs) {
+ var total : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ total = total + train_step(xs[c * 2 .. c * 2 + 2], ts[c .. c + 1], state, &hidden, &delta, &output, 0.05, cfg);
+ c = c + 1;
+ }
+ if (e == 0) {
+ first = total;
+ }
+ last = total;
+ e = e + 1;
+ }
+ if (which == 0) {
+ return first;
+ }
+ if (which == 1) {
+ return last;
+ }
+ forward(xs[(which - 2) * 2 .. (which - 2) * 2 + 2], state, &hidden, &output, cfg);
+ return output[0];
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // 784 -> 128 -> 10: exactly the 10 outputs are written, the 11th slot is untouched
test forward_output_size
- given input = create_test_input(MNIST_INPUT_SIZE)
- when output = create_test_output(MNIST_OUTPUT_SIZE)
- and state = create_test_state(MNIST_INPUT_SIZE, DEFAULT_HIDDEN_SIZE, MNIST_OUTPUT_SIZE)
- and config = MLPConfig { input_size: MNIST_INPUT_SIZE, hidden_size: DEFAULT_HIDDEN_SIZE, output_size: MNIST_OUTPUT_SIZE }
- when forward(input, state, output, config)
- then output.size() == MNIST_OUTPUT_SIZE
+ given written = mnist_forward(2, 0)
+ when spare = mnist_forward(0, 10)
+ then written == 10.0 and spare == -99.0
+ // The hidden layer is relu, so no hidden unit is negative
test forward_relu_activation
- given input = create_test_input(MNIST_INPUT_SIZE)
- when output = create_test_output(MNIST_OUTPUT_SIZE)
- and state = create_test_state(MNIST_INPUT_SIZE, DEFAULT_HIDDEN_SIZE, MNIST_OUTPUT_SIZE)
- and config = MLPConfig { input_size: MNIST_INPUT_SIZE, hidden_size: DEFAULT_HIDDEN_SIZE, output_size: MNIST_OUTPUT_SIZE }
- when forward(input, state, output, config)
- then output[0] >= 0.0 // ReLU activation should produce non-negative values
+ given min_hidden = mnist_forward(1, 0)
+ then min_hidden >= 0.0
+ // init writes every one of the 784 * 128 first-layer weights, each within sqrt(2 / 784) = 0.05051
test init_allocation_sizes
- given state = MLPState { w1: [], b1: [], w2: [], b2: [] }
- and config = MLPConfig { input_size: MNIST_INPUT_SIZE, hidden_size: DEFAULT_HIDDEN_SIZE, output_size: MNIST_OUTPUT_SIZE }
- when init(state, config)
- then state.w1.size() == MNIST_INPUT_SIZE * DEFAULT_HIDDEN_SIZE
- and state.b1.size() == DEFAULT_HIDDEN_SIZE
- and state.w2.size() == DEFAULT_HIDDEN_SIZE * MNIST_OUTPUT_SIZE
- and state.b2.size() == MNIST_OUTPUT_SIZE
+ given written = mnist_init(0)
+ when largest = mnist_init(3)
+ then written == 100352.0 and largest <= 0.0506 and largest > 0.0
test init_bias_initialization
- given state = MLPState { w1: [], b1: [], w2: [], b2: [] }
- and config = MLPConfig { input_size: MNIST_INPUT_SIZE, hidden_size: DEFAULT_HIDDEN_SIZE, output_size: MNIST_OUTPUT_SIZE }
- when init(state, config)
- then state.b1[0] == 0.0 // Biases should be initialized to zero
- and state.b2[0] == 0.0 // Biases should be initialized to zero
+ given b1 = mnist_init(1)
+ when b2 = mnist_init(2)
+ then b1 == 0.0 and b2 == 0.0
+
+ // Backpropagation learns XOR: the loss falls by more than 100x, and each output is within 0.1
+ test train_step_learns_xor
+ given first = xor(2000, 0)
+ when last = xor(2000, 1)
+ then last < first / 100.0 and @abs(xor(2000, 2)) < 0.1 and @abs(xor(2000, 3) - 1.0) < 0.1 and @abs(xor(2000, 4) - 1.0) < 0.1 and @abs(xor(2000, 5)) < 0.1
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // input_size == MNIST_INPUT_SIZE (784)
invariant mlp_constraint_0
- given input = valid_input()
- then true // input_size == MNIST_INPUT_SIZE (784)
+ assert mnist().input_size == 784
+ // output_size == MNIST_OUTPUT_SIZE (10)
invariant mlp_constraint_1
- given input = valid_input()
- then true // output_size == MNIST_OUTPUT_SIZE (10)
+ assert mnist().output_size == 10
+ // hidden_size > 0
invariant mlp_constraint_2
- given input = valid_input()
- then true // hidden_size > 0
+ assert mnist().hidden_size > 0
+ // w1.size() == input_size * hidden_size
invariant mlp_constraint_3
- given input = valid_input()
- then true // w1.size() == input_size * hidden_size
+ assert mnist().input_size * mnist().hidden_size == 100352
+ // w2.size() == hidden_size * output_size
invariant mlp_constraint_4
- given input = valid_input()
- then true // w2.size() == hidden_size * output_size
+ assert mnist().hidden_size * mnist().output_size == 1280
+ // b1.size() == hidden_size
invariant mlp_constraint_5
- given input = valid_input()
- then true // b1.size() == hidden_size
+ assert DEFAULT_HIDDEN_SIZE == 128
+ // b2.size() == output_size
invariant mlp_constraint_6
- given input = valid_input()
- then true // b2.size() == output_size
-
+ assert MNIST_OUTPUT_SIZE == 10
diff --git a/apps/website/public/t27/files/specs/ml/recurrent/attention_mechanism.t27 b/apps/website/public/t27/files/specs/ml/recurrent/attention_mechanism.t27
index 393f399ddb..48dce23e6c 100644
--- a/apps/website/public/t27/files/specs/ml/recurrent/attention_mechanism.t27
+++ b/apps/website/public/t27/files/specs/ml/recurrent/attention_mechanism.t27
@@ -2,6 +2,15 @@
// t27/specs/
// Memory-efficient attention for long sequences | φ² + 1/φ² = 3 | TRINITY
+// #8980: redeclared `var i` in one scope four times, called `@infinity()` and
+// `@random()` (not Zig), and indexed every row as head_dim wide inside d_model-wide
+// rows. backward also skipped the softmax Jacobian: it pushed dL/dweights straight
+// into the scores. It is now multi-head scaled dot-product attention over seq x
+// d_model rows. Head h reads columns h*hd .. (h+1)*hd, the weights are
+// heads x seq x seq, and causal masking is optional. backward is the exact
+// gradient, checked against finite differences. Dropout is carried in the config
+// and not applied: this is the inference computation.
+
module Attention;
use base::types;
use math::constants;
@@ -10,7 +19,7 @@ module Attention;
// 1. Constants
// ═══════════════════════════════════════════════════════════
- const SCALE_FACTOR : f32 = 0.0;
+ const NEG_INF : f32 = -1.0e30;
// ═══════════════════════════════════════════════════════════
// 2. Types
@@ -32,232 +41,267 @@ module Attention;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(query: []const f32, key: []const f32, value: []const f32, output: []f32, weights: []f32, config: AttentionConfig) → void
+ fn seq_of(query: []const f32, config: AttentionConfig) -> usize {
+ return query.len / config.d_model;
+ }
+
+ fn scale_of(config: AttentionConfig) -> f32 {
+ return 1.0 / sqrt(@as(f32, @floatFromInt(config.d_model / config.num_heads)));
+ }
+
fn forward(query: []const f32, key: []const f32, value: []const f32, output: []f32, weights: []f32, config: AttentionConfig) -> void {
- // Compute attention scores: query · key^T
- const seq_len : u32 = @intCast(query.len / config.d_model);
- const head_dim : u32 = config.d_model / config.num_heads;
-
- // Scale query vectors
- const scaled_query = []f32(query.len);
- var i : u32 = 0;
- while (i < query.len) : (i += 1) {
- scaled_query[i] = query[i] * @floatFromInt(1) / @sqrt(@floatFromInt(head_dim));
- }
-
- // Compute attention weights
- var i : u32 = 0;
- while (i < seq_len) : (i += 1) {
- var j : u32 = 0;
- while (j < seq_len) : (j += 1) {
- var k : u32 = 0;
- var score : f32 = 0.0;
- while (k < head_dim) : (k += 1) {
- score += scaled_query[i * head_dim + k] * key[j * head_dim + k];
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const row = weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var max_s : f32 = NEG_INF;
+ var j : usize = 0;
+ while (j < n) {
+ var score : f32 = NEG_INF;
+ if (!(config.causal and j > i)) {
+ score = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ score = score + query[i * d + h * hd + k] * key[j * d + h * hd + k];
+ k = k + 1;
+ }
+ score = score * scale;
+ }
+ row[j] = score;
+ max_s = @max(max_s, score);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ j = 0;
+ while (j < n) {
+ row[j] = row[j] / sum;
+ j = j + 1;
}
-
- // Apply causal masking if enabled
- if (config.causal and j > i) {
- score = -@infinity();
+ var k : usize = 0;
+ while (k < hd) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ acc = acc + row[j] * value[j * d + h * hd + k];
+ j = j + 1;
+ }
+ output[i * d + h * hd + k] = acc;
+ k = k + 1;
}
-
- weights[i * seq_len + j] = score;
+ i = i + 1;
}
+ h = h + 1;
}
-
- // Apply softmax to get attention probabilities
- var i : u32 = 0;
- while (i < seq_len) : (i += 1) {
- var max_score : f32 = -@infinity();
- var j : u32 = 0;
- while (j < seq_len) : (j += 1) {
- max_score = @max(max_score, weights[i * seq_len + j]);
- }
-
- j = 0;
- var sum_exp : f32 = 0.0;
- while (j < seq_len) : (j += 1) {
- weights[i * seq_len + j] = @exp(weights[i * seq_len + j] - max_score);
- sum_exp += weights[i * seq_len + j];
- }
-
- j = 0;
- while (j < seq_len) : (j += 1) {
- weights[i * seq_len + j] /= sum_exp;
- }
+ }
+
+ // dV_j += P_ij dO_i; dP_ij = dO_i . V_j; dS_ij = P_ij (dP_ij - sum_k P_ik dP_ik);
+ // dQ_i += scale dS_ij K_j; dK_j += scale dS_ij Q_i (per head; masked P_ij are 0)
+ fn backward(grad_output: []const f32, query: []const f32, key: []const f32, value: []const f32, attn_weights: []f32, grad_query: []f32, grad_key: []f32, grad_value: []f32, config: AttentionConfig) -> void {
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var x : usize = 0;
+ while (x < n * d) {
+ grad_query[x] = 0.0;
+ grad_key[x] = 0.0;
+ grad_value[x] = 0.0;
+ x = x + 1;
}
-
- // Compute weighted sum of values
- var i : u32 = 0;
- while (i < seq_len) : (i += 1) {
- var k : u32 = 0;
- while (k < head_dim) : (k += 1) {
- var j : u32 = 0;
- var weighted_sum : f32 = 0.0;
- while (j < seq_len) : (j += 1) {
- weighted_sum += weights[i * seq_len + j] * value[j * head_dim + k];
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const p = attn_weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var row_dot : f32 = 0.0;
+ var j : usize = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ grad_value[j * d + h * hd + k] = grad_value[j * d + h * hd + k] + p[j] * grad_output[i * d + h * hd + k];
+ k = k + 1;
+ }
+ row_dot = row_dot + p[j] * dp;
+ j = j + 1;
}
- output[i * head_dim + k] = weighted_sum;
- }
- }
-
- // Apply dropout if enabled
- if (config.dropout > 0.0) {
- var i : u32 = 0;
- while (i < weights.len) : (i += 1) {
- if (@random() < config.dropout) {
- weights[i] = 0.0;
+ j = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ k = k + 1;
+ }
+ const ds = p[j] * (dp - row_dot) * scale;
+ k = 0;
+ while (k < hd) {
+ grad_query[i * d + h * hd + k] = grad_query[i * d + h * hd + k] + ds * key[j * d + h * hd + k];
+ grad_key[j * d + h * hd + k] = grad_key[j * d + h * hd + k] + ds * query[i * d + h * hd + k];
+ k = k + 1;
+ }
+ j = j + 1;
}
+ i = i + 1;
}
+ h = h + 1;
}
}
- // backward(grad_output: []const f32, query: []const f32, key: []const f32, value: []const f32, attn_weights: []f32, grad_query: []f32, grad_key: []f32, grad_value: []f32, config: AttentionConfig) → void
- fn backward(grad_output: []const f32, query: []const f32, key: []const f32, value: []const f32, attn_weights: []f32, grad_query: []f32, grad_key: []f32, grad_value: []f32, config: AttentionConfig) -> void {
- const seq_len : u32 = @intCast(query.len / config.d_model);
- const head_dim : u32 = config.d_model / config.num_heads;
-
- // Initialize gradients to zero
- var i : u32 = 0;
- while (i < grad_query.len) : (i += 1) {
- grad_query[i] = 0.0;
+ fn cfg(causal: bool, dropout: f32) -> AttentionConfig {
+ return AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = dropout, .causal = causal };
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads (columns 0-1 and 2-3); q / k / v scaled down so the
+ // softmax is not saturated. which: 0 = output, 1 = weights.
+ fn fixture(q: []f32, k: []f32, v: []f32) -> void {
+ var x : usize = 0;
+ while (x < 12) {
+ q[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ k[x] = @as(f32, @floatFromInt(x / 4 + 1)) / 4.0 - @as(f32, @floatFromInt(x % 2)) / 3.0;
+ v[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
}
- i = 0;
- while (i < grad_key.len) : (i += 1) {
- grad_key[i] = 0.0;
+ }
+
+ fn forward_at(causal: bool, which: u8, index: usize) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = [9.0] ** 12;
+ var w : [18]f32 = [9.0] ** 18;
+ forward(&q, &k, &v, &out, &w, cfg(causal, 0.0));
+ if (which == 0) {
+ return out[index];
}
- i = 0;
- while (i < grad_value.len) : (i += 1) {
- grad_value[i] = 0.0;
+ return w[index];
+ }
+
+ // L = sum_x g_x O_x with g_x = (x + 1) / 10, one entry of q, k or v moved by delta
+ // (which: 0 = q, 1 = k, 2 = v)
+ fn loss(which: u8, index: usize, delta: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ if (which == 0) {
+ q[index] = q[index] + delta;
}
-
- // Compute gradients for value: grad_output · attention_weights^T
- var i : u32 = 0;
- while (i < seq_len) : (i += 1) {
- var k : u32 = 0;
- while (k < head_dim) : (k += 1) {
- var j : u32 = 0;
- while (j < seq_len) : (j += 1) {
- grad_value[j * head_dim + k] += grad_output[i * head_dim + k] * attn_weights[i * seq_len + j];
- }
- }
+ if (which == 1) {
+ k[index] = k[index] + delta;
}
-
- // Compute gradients for attention weights: grad_output · value^T
- var grad_weights = []f32(attn_weights.len);
- var i : u32 = 0;
- while (i < seq_len) : (i += 1) {
- var j : u32 = 0;
- while (j < seq_len) : (j += 1) {
- var k : u32 = 0;
- var grad_attn : f32 = 0.0;
- while (k < head_dim) : (k += 1) {
- grad_attn += grad_output[i * head_dim + k] * value[j * head_dim + k];
- }
- grad_weights[i * seq_len + j] = grad_attn;
- }
+ if (which == 2) {
+ v[index] = v[index] + delta;
}
-
- // Compute gradients for query and key through attention weights
- var i : u32 = 0;
- while (i < seq_len) : (i += 1) {
- var j : u32 = 0;
- while (j < seq_len) : (j += 1) {
- if (config.causal and j > i) {
- continue; // Skip causal positions
- }
-
- var k : u32 = 0;
- while (k < head_dim) : (k += 1) {
- // Gradient for query
- grad_query[i * head_dim + k] += grad_weights[i * seq_len + j] * key[j * head_dim + k];
-
- // Gradient for key
- grad_key[j * head_dim + k] += grad_weights[i * seq_len + j] * query[i * head_dim + k];
- }
- }
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, 0.0));
+ var total : f32 = 0.0;
+ var x : usize = 0;
+ while (x < 12) {
+ total = total + @as(f32, @floatFromInt(x + 1)) / 10.0 * out[x];
+ x = x + 1;
}
-
- // Apply scaling factor to gradients
- const scale : f32 = @floatFromInt(1) / @sqrt(@floatFromInt(head_dim));
- var i : u32 = 0;
- while (i < grad_query.len) : (i += 1) {
- grad_query[i] *= scale;
+ return total;
+ }
+
+ fn grad(which: u8, index: usize, dropout: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, dropout));
+ var g : [12]f32 = undefined;
+ var x : usize = 0;
+ while (x < 12) {
+ g[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
}
- i = 0;
- while (i < grad_key.len) : (i += 1) {
- grad_key[i] *= scale;
+ var gq : [12]f32 = undefined;
+ var gk : [12]f32 = undefined;
+ var gv : [12]f32 = undefined;
+ backward(&g, &q, &k, &v, &w, &gq, &gk, &gv, cfg(false, dropout));
+ if (which == 0) {
+ return gq[index];
}
+ if (which == 1) {
+ return gk[index];
+ }
+ return gv[index];
+ }
+
+ fn slope(which: u8, index: usize) -> f32 {
+ return (loss(which, index, 0.001) - loss(which, index, -0.001)) / 0.002;
+ }
+
+ fn valid_input() -> u32 {
+ return cfg(false, 0.0).d_model % cfg(false, 0.0).num_heads;
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // Each weight row is a distribution, and each output is a weighted mean of the values
+ // (so it lies between the smallest and largest value in its column)
test forward_basic_case {
- var query: [12]f32 = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0];
- var key: [12]f32 = [1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0];
- var value: [12]f32 = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2];
- var config = AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = 0.0, .causal = false };
- var output: [12]f32 = [0.0] ** 12;
- var weights: [9]f32 = [0.0] ** 9;
- forward(&query, &key, &value, &output, &weights, config);
- assert(output[0] + output[1] + output[2] + output[3] > 0.0); // computed a meaningful output
+ const row_sum = forward_at(false, 1, 0) + forward_at(false, 1, 1) + forward_at(false, 1, 2);
+ assert(@abs(row_sum - 1.0) < 0.000001);
+ assert(forward_at(false, 0, 0) + forward_at(false, 0, 1) + forward_at(false, 0, 2) + forward_at(false, 0, 3) > 0.0); // computed a meaningful output
+ assert(forward_at(false, 0, 0) > 0.1 and forward_at(false, 0, 0) < 0.9);
}
+ // dQ, dK and dV against central differences of L = g . O
test backward_basic_case {
- var grad_output: [12]f32 = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2];
- var query: [12]f32 = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0];
- var key: [12]f32 = [1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0];
- var value: [12]f32 = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2];
- var attn_weights: [9]f32 = [0.3, 0.3, 0.4, 0.3, 0.3, 0.4, 0.4, 0.4, 0.2];
- var config = AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = 0.0, .causal = false };
- var grad_query: [12]f32 = [0.0] ** 12;
- var grad_key: [12]f32 = [0.0] ** 12;
- var grad_value: [12]f32 = [0.0] ** 12;
- backward(&grad_output, &query, &key, &value, &attn_weights, &grad_query, &grad_key, &grad_value, config);
- assert(grad_query[0] + grad_query[1] + grad_query[2] + grad_query[3] != 0.0); // computed gradients
+ assert(@abs(grad(0, 0, 0.0) - slope(0, 0)) < 0.001);
+ assert(@abs(grad(0, 7, 0.0) - slope(0, 7)) < 0.001);
+ assert(@abs(grad(1, 5, 0.0) - slope(1, 5)) < 0.001);
+ assert(@abs(grad(2, 10, 0.0) - slope(2, 10)) < 0.001);
+ assert(grad(0, 0, 0.0) + grad(0, 1, 0.0) + grad(0, 2, 0.0) + grad(0, 3, 0.0) != 0.0); // computed gradients
}
+ // Position 0 attends only to itself, position 1 to positions 0 and 1
test forward_causal_masking {
- var query: [12]f32 = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0];
- var key: [12]f32 = [1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0];
- var value: [12]f32 = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2];
- var config = AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = 0.0, .causal = true };
- var output: [12]f32 = [0.0] ** 12;
- var weights: [9]f32 = [0.0] ** 9;
- forward(&query, &key, &value, &output, &weights, config);
- // A position may not attend to a later one, so its weight there stays far below 1.
- assert(weights[1 * 3 + 2] < 0.999);
+ assert(forward_at(true, 1, 0) == 1.0);
+ assert(forward_at(true, 1, 1) == 0.0 and forward_at(true, 1, 2) == 0.0);
+ assert(forward_at(true, 1, 1 * 3 + 2) < 0.999);
+ assert(forward_at(true, 1, 1 * 3 + 2) == 0.0);
+ assert(forward_at(true, 0, 0) == 0.1);
}
+ // Dropout is not applied here (inference), so it leaves the gradients unchanged
test backward_dropout {
- var grad_output: [12]f32 = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2];
- var query: [12]f32 = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0];
- var key: [12]f32 = [1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0];
- var value: [12]f32 = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2];
- var attn_weights: [9]f32 = [0.3, 0.3, 0.4, 0.3, 0.3, 0.4, 0.4, 0.4, 0.2];
- var config = AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = 0.5, .causal = false };
- var grad_query: [12]f32 = [0.0] ** 12;
- var grad_key: [12]f32 = [0.0] ** 12;
- var grad_value: [12]f32 = [0.0] ** 12;
- backward(&grad_output, &query, &key, &value, &attn_weights, &grad_query, &grad_key, &grad_value, config);
- assert(grad_value.len == 12);
+ assert(grad(2, 3, 0.5) == grad(2, 3, 0.0));
+ assert(grad(0, 11, 0.5) == grad(0, 11, 0.0));
}
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // d_model % num_heads == 0
invariant attention_constraint_0
- given input = valid_input()
- then true // d_model % num_heads == 0
+ assert valid_input() == 0
+ // 0 <= dropout < 1
invariant attention_constraint_1
- given input = valid_input()
- then true // 0 <= dropout < 1
+ assert cfg(false, 0.5).dropout >= 0.0 and cfg(false, 0.5).dropout < 1.0
+ // query.shape == key.shape == value.shape: seq is read from the query
invariant attention_constraint_2
- given input = valid_input()
- then true // query.shape == key.shape == value.shape
-
+ assert cfg(false, 0.0).d_model == 4
diff --git a/apps/website/public/t27/files/specs/ml/recurrent/lstm_single.t27 b/apps/website/public/t27/files/specs/ml/recurrent/lstm_single.t27
index 0e216bac38..bc16437835 100644
--- a/apps/website/public/t27/files/specs/ml/recurrent/lstm_single.t27
+++ b/apps/website/public/t27/files/specs/ml/recurrent/lstm_single.t27
@@ -2,6 +2,15 @@
// t27/specs/
// LSTM solves vanishing gradient problem | φ² + 1/φ² = 3 | TRINITY
+// #8980: called `@sigmoid` and `@tanh`, which Zig does not have; each gate weight
+// was one number per INPUT, indexed `i % hidden_size`, so it was not an LSTM; and
+// every test asserted `true`. Now each gate has its own matrices, W_x
+// (hidden x input) and W_h (hidden x hidden), and a bias:
+// i = s(z_i), f = s(z_f), g = tanh(z_g), o = s(z_o), z = W_x x + W_h h + b
+// c' = f c + i g, h' = o tanh(c')
+// backward gives every gradient of one step, and the tests check it against finite
+// differences.
+
module LstmCell;
use base::types;
use math::constants;
@@ -45,190 +54,272 @@ module LstmCell;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(input: []const f32, state: LSTMState, weights: LSTMWeights, output_state: LSTMState, config: LSTMConfig) -> void
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Pre-activation of one gate for hidden unit u: W_x[u] . x + W_h[u] . h + b[u]
+ fn pre(w_x: []const f32, w_h: []const f32, b: []const f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig) -> f32 {
+ var z : f32 = b[u];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + w_x[u * config.input_size + j] * x[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + w_h[u * config.hidden_size + k] * h[k];
+ k = k + 1;
+ }
+ return z;
+ }
+
fn forward(input: []const f32, state: LSTMState, weights: LSTMWeights, output_state: LSTMState, config: LSTMConfig) -> void {
- // LSTM forward pass implementation
- // Compute input, forget, output, and cell gates
- for (0..config.input_size) |i| {
- var input_val:f32 = input[i];
-
- // Input gate: i = σ(W_ii * x + W_hi * h + b_i)
- var input_gate:f32 = @sigmoid(weights.W_ii[i] * input_val + weights.W_hi[i] * state.h[i % config.hidden_size] + weights.b_i[i % config.hidden_size]);
-
- // Forget gate: f = σ(W_if * x + W_hf * h + b_f)
- var forget_gate:f32 = @sigmoid(weights.W_if[i] * input_val + weights.W_hf[i] * state.h[i % config.hidden_size] + weights.b_f[i % config.hidden_size]);
-
- // Output gate: o = σ(W_io * x + W_ho * h + b_o)
- var output_gate:f32 = @sigmoid(weights.W_io[i] * input_val + weights.W_ho[i] * state.h[i % config.hidden_size] + weights.b_o[i % config.hidden_size]);
-
- // Cell gate candidate: g = tanh(W_ig * x + W_hg * h + b_g)
- var cell_candidate:f32 = @tanh(weights.W_ig[i] * input_val + weights.W_hg[i] * state.h[i % config.hidden_size] + weights.b_g[i % config.hidden_size]);
-
- // Update cell state: c = f * c + i * g
- output_state.c[i % config.hidden_size] = forget_gate * state.c[i % config.hidden_size] + input_gate * cell_candidate;
-
- // Update hidden state: h = o * tanh(c)
- output_state.h[i % config.hidden_size] = output_gate * @tanh(output_state.c[i % config.hidden_size]);
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ output_state.c[u] = fg * state.c[u] + ig * gg;
+ output_state.h[u] = og * tanh(output_state.c[u]);
+ u = u + 1;
}
}
- // backward(grad_output: []const f32, grad_cell: []const f32, input: []const f32, state: LSTMState, weights: LSTMWeights, grad_input: []f32, grad_state: LSTMState, grad_weights: LSTMWeights) -> void
- fn backward(grad_output: []const f32, grad_cell: []const f32, input: []const f32, state: LSTMState, weights: LSTMWeights, grad_input: []f32, grad_state: LSTMState, grad_weights: LSTMWeights) -> void {
- // LSTM backward pass implementation
- var config:LSTMConfig = LSTMConfig{
- .input_size = input.size(),
- .hidden_size = state.h.size(),
+ // Adds dz * (x, h, 1) into one gate's weight gradients, and dz * W into grad_input / grad_state.h
+ fn spread(dz: f32, w_x: []const f32, w_h: []const f32, gw_x: []f32, gw_h: []f32, gb: []f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig, grad_input: []f32, grad_h: []f32) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ gw_x[u * config.input_size + j] = gw_x[u * config.input_size + j] + dz * x[j];
+ grad_input[j] = grad_input[j] + dz * w_x[u * config.input_size + j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ gw_h[u * config.hidden_size + k] = gw_h[u * config.hidden_size + k] + dz * h[k];
+ grad_h[k] = grad_h[k] + dz * w_h[u * config.hidden_size + k];
+ k = k + 1;
+ }
+ gb[u] = gb[u] + dz;
+ }
+
+ // One step back. grad_output = dL/dh', grad_cell = dL/dc' from later steps. Writes
+ // dL/dx, dL/dh, dL/dc and ADDS the weight gradients into grad_weights.
+ // do = dh' tanh(c'); dc = dh' o (1 - tanh^2 c') + grad_cell
+ // dz_o = do o (1 - o), dz_f = dc c f (1 - f), dz_i = dc g i (1 - i), dz_g = dc i (1 - g^2)
+ // dL/dc = dc f
+ fn backward(grad_output: []const f32, grad_cell: []const f32, input: []const f32, state: LSTMState, weights: LSTMWeights, config: LSTMConfig, grad_input: []f32, grad_state: LSTMState, grad_weights: LSTMWeights) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ grad_input[j] = 0.0;
+ j = j + 1;
+ }
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ grad_state.h[u] = 0.0;
+ u = u + 1;
+ }
+ u = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ const c_new = fg * state.c[u] + ig * gg;
+ const tc = tanh(c_new);
+ const dc = grad_output[u] * og * (1.0 - tc * tc) + grad_cell[u];
+ const dz_o = grad_output[u] * tc * og * (1.0 - og);
+ const dz_f = dc * state.c[u] * fg * (1.0 - fg);
+ const dz_i = dc * gg * ig * (1.0 - ig);
+ const dz_g = dc * ig * (1.0 - gg * gg);
+ grad_state.c[u] = dc * fg;
+ spread(dz_i, weights.W_ii, weights.W_hi, grad_weights.W_ii, grad_weights.W_hi, grad_weights.b_i, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_f, weights.W_if, weights.W_hf, grad_weights.W_if, grad_weights.W_hf, grad_weights.b_f, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_g, weights.W_ig, weights.W_hg, grad_weights.W_ig, grad_weights.W_hg, grad_weights.b_g, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_o, weights.W_io, weights.W_ho, grad_weights.W_io, grad_weights.W_ho, grad_weights.b_o, input, state.h, u, config, grad_input, grad_state.h);
+ u = u + 1;
+ }
+ }
+
+ // The fixture: input 3, hidden 2. All 48 weights live in one buffer, per gate
+ // W_x (6), W_h (4), b (2), in the gate order i, f, g, o.
+ fn cfg() -> LSTMConfig {
+ return LSTMConfig{ .input_size = 3, .hidden_size = 2 };
+ }
+
+ fn weights_of(w: []f32) -> LSTMWeights {
+ return LSTMWeights{
+ .W_ii = w[0..6], .W_hi = w[6..10], .b_i = w[10..12],
+ .W_if = w[12..18], .W_hf = w[18..22], .b_f = w[22..24],
+ .W_ig = w[24..30], .W_hg = w[30..34], .b_g = w[34..36],
+ .W_io = w[36..42], .W_ho = w[42..46], .b_o = w[46..48],
};
-
- // Initialize gradients to zero
- for (0..grad_input.size()) |i| {
- grad_input[i] = 0.0;
- }
-
- for (0..grad_state.h.size()) |i| {
- grad_state.h[i] = 0.0;
- grad_state.c[i] = 0.0;
- }
-
- // Backward pass through LSTM cell
- for (0..config.input_size) |i| {
- var input_val:f32 = input[i];
-
- // Compute forward pass gates (needed for backward pass)
- var input_gate:f32 = @sigmoid(weights.W_ii[i] * input_val + weights.W_hi[i] * state.h[i % config.hidden_size] + weights.b_i[i % config.hidden_size]);
- var forget_gate:f32 = @sigmoid(weights.W_if[i] * input_val + weights.W_hf[i] * state.h[i % config.hidden_size] + weights.b_f[i % config.hidden_size]);
- var output_gate:f32 = @sigmoid(weights.W_io[i] * input_val + weights.W_ho[i] * state.h[i % config.hidden_size] + weights.b_o[i % config.hidden_size]);
- var cell_candidate:f32 = @tanh(weights.W_ig[i] * input_val + weights.W_hg[i] * state.h[i % config.hidden_size] + weights.b_g[i % config.hidden_size]);
-
- // Compute derivatives
- var cell_derivative:f32 = 1.0 - @tanh(state.c[i % config.hidden_size]) * @tanh(state.c[i % config.hidden_size]);
- var cell_candidate_derivative:f32 = 1.0 - cell_candidate * cell_candidate;
-
- // Backpropagate through output gate
- var output_grad:f32 = grad_output[i % config.hidden_size] * @tanh(state.c[i % config.hidden_size]) * output_gate * (1.0 - output_gate);
-
- // Backpropagate through cell state
- var cell_grad:f32 = grad_output[i % config.hidden_size] * output_gate * cell_derivative + grad_cell[i % config.hidden_size];
-
- // Backpropagate through forget gate
- var forget_grad:f32 = cell_grad * state.c[i % config.hidden_size] * forget_gate * (1.0 - forget_gate);
-
- // Backpropagate through input gate
- var input_grad:f32 = cell_grad * cell_candidate * input_gate * (1.0 - input_gate);
-
- // Accumulate gradients for input
- grad_input[i] += input_grad * weights.W_ii[i] + forget_grad * weights.W_if[i] + output_grad * weights.W_io[i] + cell_grad * cell_candidate_derivative * weights.W_ig[i];
-
- // Accumulate gradients for hidden state
- grad_state.h[i % config.hidden_size] += input_grad * weights.W_hi[i] + forget_grad * weights.W_hf[i] + output_grad * weights.W_ho[i] + cell_grad * cell_candidate_derivative * weights.W_hg[i];
-
- // Accumulate gradients for weights (simplified - in practice we'd sum over batch)
- grad_weights.b_i[i % config.hidden_size] += input_grad;
- grad_weights.b_f[i % config.hidden_size] += forget_grad;
- grad_weights.b_o[i % config.hidden_size] += output_grad;
- grad_weights.b_g[i % config.hidden_size] += cell_grad * cell_candidate_derivative;
+ }
+
+ // Deterministic weights in [-0.5, 0.5]; the forget biases start at FORGET_BIAS_INIT
+ fn fill(w: []f32) -> void {
+ var k : usize = 0;
+ while (k < 48) {
+ w[k] = @as(f32, @floatFromInt((k * 7 + 3) % 11)) / 10.0 - 0.5;
+ k = k + 1;
+ }
+ w[22] = FORGET_BIAS_INIT;
+ w[23] = FORGET_BIAS_INIT;
+ }
+
+ // L = 0.5 h'_0 - 0.3 h'_1 + 0.2 c'_0 + 0.1 c'_1 after one step, with one input, one h or
+ // one c entry moved by `delta` (which: 0 = none, 1 + j = x_j, 4 + k = h_k, 6 + k = c_k,
+ // 8 = b_f[1])
+ fn loss(which: usize, delta: f32) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ var x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ if (which >= 1 and which <= 3) {
+ x[which - 1] = x[which - 1] + delta;
+ }
+ if (which >= 4 and which <= 5) {
+ h[which - 4] = h[which - 4] + delta;
+ }
+ if (which >= 6 and which <= 7) {
+ c[which - 6] = c[which - 6] + delta;
}
+ if (which == 8) {
+ w[23] = w[23] + delta;
+ }
+ var h_out : [2]f32 = [0.0] ** 2;
+ var c_out : [2]f32 = [0.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ return 0.5 * h_out[0] - 0.3 * h_out[1] + 0.2 * c_out[0] + 0.1 * c_out[1];
+ }
+
+ // The analytic gradient of that loss, same `which` numbering
+ fn grad(which: usize) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const grad_output : [2]f32 = [0.5, -0.3];
+ const grad_cell : [2]f32 = [0.2, 0.1];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&grad_output, &grad_cell, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ if (which >= 1 and which <= 3) {
+ return gx[which - 1];
+ }
+ if (which >= 4 and which <= 5) {
+ return gh[which - 4];
+ }
+ if (which >= 6 and which <= 7) {
+ return gc[which - 6];
+ }
+ return gw[23];
+ }
+
+ // Sum of |every gradient| when grad_output and grad_cell are both 0
+ fn zero_upstream_total() -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const zeros : [2]f32 = [0.0, 0.0];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&zeros, &zeros, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ var total : f32 = @abs(gx[0]) + @abs(gx[1]) + @abs(gx[2]) + @abs(gh[0]) + @abs(gh[1]) + @abs(gc[0]) + @abs(gc[1]);
+ var k : usize = 0;
+ while (k < 48) {
+ total = total + @abs(gw[k]);
+ k = k + 1;
+ }
+ return total;
+ }
+
+ fn slope(which: usize) -> f32 {
+ return (loss(which, 0.001) - loss(which, -0.001)) / 0.002;
+ }
+
+ // which: 0 = h'_0, 1 = c'_0 for a 1-unit cell with every weight 0 and the given biases
+ fn bias_only(b_i: f32, b_f: f32, b_g: f32, b_o: f32, c_prev: f32, which: u8) -> f32 {
+ var w : [48]f32 = [0.0] ** 48;
+ w[10] = b_i;
+ w[22] = b_f;
+ w[34] = b_g;
+ w[46] = b_o;
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.0, 0.0];
+ var c : [2]f32 = [c_prev, 0.0];
+ var h_out : [2]f32 = [9.0] ** 2;
+ var c_out : [2]f32 = [9.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ if (which == 0) {
+ return h_out[0];
+ }
+ return c_out[0];
+ }
+
+ fn valid_input() -> usize {
+ return cfg().input_size;
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // All weights 0: every gate is s(0) = 0.5 or tanh(0) = 0, so c' = 0.5 c and h' = 0.5 tanh(c')
test forward_basic_case
- given input = [1.0f32, 2.0f32, 3.0f32]
- when result = forward(input)
- then true // Forward pass completes without error
+ given c_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 1)
+ when h_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 0)
+ then @abs(c_new - 0.4) < 0.000001 and @abs(h_new - 0.5 * tanh(c_new)) < 0.000001 and @abs(h_new - 0.1899745) < 0.000001
- test backward_basic_case
- given grad_output = [0.5f32, -0.3f32, 0.2f32]
- when result = backward(grad_output)
- then true // Backward pass completes without error
+ // A closed forget gate (b_f = -30) erases the old cell; an open one (b_f = 30) keeps it
+ test forward_basic_lstm
+ given erased = bias_only(0.0, -30.0, 0.0, 0.0, 0.8, 1)
+ when kept = bias_only(-30.0, 30.0, 0.0, 0.0, 0.8, 1)
+ then @abs(erased) < 0.000001 and @abs(kept - 0.8) < 0.000001
+ // A closed input gate and forget gate leave c' = 0, so h' = 0 whatever the output gate does
test forward_with_empty_input
- given input = []f32
- when result = forward(input)
- then true // Handles empty input gracefully
-
- test backward_with_empty_grad
- given grad_output = []f32
- when result = backward(grad_output)
- then true // Handles empty gradient input gracefully
+ given h_new = bias_only(-30.0, -30.0, 1.0, 30.0, 0.8, 0)
+ then @abs(h_new) < 0.000001
- test forward_basic_lstm
- given
- input = [1.0f32, 2.0f32, 3.0f32]
- state = LSTMState{.h = [0.1f32, 0.2f32], .c = [0.3f32, 0.4f32]}
- weights = LSTMWeights{
- .W_ii = [0.1f32, 0.2f32, 0.3f32],
- .W_hi = [0.4f32, 0.5f32],
- .b_i = [0.6f32, 0.7f32],
- .W_if = [0.8f32, 0.9f32, 1.0f32],
- .W_hf = [1.1f32, 1.2f32],
- .b_f = [1.3f32, 1.4f32],
- .W_ig = [1.5f32, 1.6f32, 1.7f32],
- .W_hg = [1.8f32, 1.9f32],
- .b_g = [2.0f32, 2.1f32],
- .W_io = [2.2f32, 2.3f32, 2.4f32],
- .W_ho = [2.5f32, 2.6f32],
- .b_o = [2.7f32, 2.8f32],
- }
- output_state = LSTMState{.h = [0.0f32, 0.0f32], .c = [0.0f32, 0.0f32]}
- config = LSTMConfig{.input_size = 3, .hidden_size = 2}
- when result = forward(input, state, weights, output_state, config)
- then true // Forward pass completes without error
+ // dL/dx_j against central differences
+ test backward_basic_case
+ given g0 = grad(1)
+ when g2 = grad(3)
+ then @abs(g0 - slope(1)) < 0.0005 and @abs(grad(2) - slope(2)) < 0.0005 and @abs(g2 - slope(3)) < 0.0005
+ // dL/dh_k, dL/dc_k and dL/db_f[1] against central differences
test backward_basic_lstm
- given
- grad_output = [0.5f32, -0.3f32]
- grad_cell = [0.2f32, 0.1f32]
- input = [1.0f32, 2.0f32, 3.0f32]
- state = LSTMState{.h = [0.1f32, 0.2f32], .c = [0.3f32, 0.4f32]}
- weights = LSTMWeights{
- .W_ii = [0.1f32, 0.2f32, 0.3f32],
- .W_hi = [0.4f32, 0.5f32],
- .b_i = [0.6f32, 0.7f32],
- .W_if = [0.8f32, 0.9f32, 1.0f32],
- .W_hf = [1.1f32, 1.2f32],
- .b_f = [1.3f32, 1.4f32],
- .W_ig = [1.5f32, 1.6f32, 1.7f32],
- .W_hg = [1.8f32, 1.9f32],
- .b_g = [2.0f32, 2.1f32],
- .W_io = [2.2f32, 2.3f32, 2.4f32],
- .W_ho = [2.5f32, 2.6f32],
- .b_o = [2.7f32, 2.8f32],
- }
- grad_input = [0.0f32, 0.0f32, 0.0f32]
- grad_state = LSTMState{.h = [0.0f32, 0.0f32], .c = [0.0f32, 0.0f32]}
- grad_weights = LSTMWeights{
- .W_ii = [0.0f32, 0.0f32, 0.0f32],
- .W_hi = [0.0f32, 0.0f32],
- .b_i = [0.0f32, 0.0f32],
- .W_if = [0.0f32, 0.0f32, 0.0f32],
- .W_hf = [0.0f32, 0.0f32],
- .b_f = [0.0f32, 0.0f32],
- .W_ig = [0.0f32, 0.0f32, 0.0f32],
- .W_hg = [0.0f32, 0.0f32],
- .b_g = [0.0f32, 0.0f32],
- .W_io = [0.0f32, 0.0f32, 0.0f32],
- .W_ho = [0.0f32, 0.0f32],
- .b_o = [0.0f32, 0.0f32],
- }
- when result = backward(grad_output, grad_cell, input, state, weights, grad_input, grad_state, grad_weights)
- then true // Backward pass completes without error
+ given gh0 = grad(4)
+ when gc1 = grad(7)
+ then @abs(gh0 - slope(4)) < 0.0005 and @abs(grad(5) - slope(5)) < 0.0005 and @abs(grad(6) - slope(6)) < 0.0005 and @abs(gc1 - slope(7)) < 0.0005 and @abs(grad(8) - slope(8)) < 0.0005
+
+ // With no upstream gradient nothing flows back
+ test backward_with_empty_grad
+ given total = zero_upstream_total()
+ then total == 0.0 and @abs(grad(1)) > 0.0
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // input.size() == config.input_size
invariant lstm_cell_constraint_0
- given input = valid_input()
- then true // input.size() == config.input_size
+ assert valid_input() == 3
+ // state.h.size() == config.hidden_size
invariant lstm_cell_constraint_1
- given input = valid_input()
- then true // state.h.size() == config.hidden_size
+ assert cfg().hidden_size == 2
+ // state.c.size() == config.hidden_size; the forget bias starts open
invariant lstm_cell_constraint_2
- given input = valid_input()
- then true // state.c.size() == config.hidden_size
-
+ assert FORGET_BIAS_INIT > 0.0
diff --git a/apps/website/public/t27/files/specs/ml/recurrent/rnn_cell.t27 b/apps/website/public/t27/files/specs/ml/recurrent/rnn_cell.t27
index 56e54d7a3f..0ad0f92fae 100644
--- a/apps/website/public/t27/files/specs/ml/recurrent/rnn_cell.t27
+++ b/apps/website/public/t27/files/specs/ml/recurrent/rnn_cell.t27
@@ -2,6 +2,12 @@
// t27/specs/
// Basic RNN recurrence equation | φ² + 1/φ² = 3 | TRINITY
+// #8980: written against undeclared `matmul`, `add`, `zeros`, `len` and `append`,
+// it applied tanh twice (h = tanh(tanh(z))), and the tests asserted
+// `result != undefined`. It is now the Elman cell,
+// h' = tanh(W_x x + W_h h + b), with W_x hidden x input and W_h hidden x hidden
+// (row-major), stepping over buffers the caller owns.
+
module RnnCell;
use base::types;
use math::constants;
@@ -37,99 +43,148 @@ module RnnCell;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward_step(input: []f32, state: RNNState, params: RNNParams) -> RNNState
- fn forward_step(input: []f32, state: RNNState, params: RNNParams) -> RNNState {
- // Basic RNN forward step implementation
- // Apply linear transformation to input
- let input_transformed: []f32 = matmul(params.w_x, input);
-
- // Apply linear transformation to hidden state
- let hidden_transformed: []f32 = matmul(params.w_h, state.hidden);
-
- // Add bias and apply activation function (tanh for cell state)
- let cell_input: []f32 = add(add(input_transformed, hidden_transformed), params.b);
- let new_cell: []f32 = tanh(cell_input);
-
- // Apply activation function (tanh) for hidden state
- let new_hidden: []f32 = tanh(new_cell);
-
- return RNNState {
- hidden: new_hidden,
- cell: new_cell,
- };
+ // new_hidden[i] = tanh(sum_j W_x[i][j] x[j] + sum_k W_h[i][k] h[k] + b[i])
+ fn forward_step(input: []const f32, hidden: []const f32, params: RNNParams, config: RNNCellConfig, new_hidden: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.hidden_size) {
+ var z : f32 = params.b[i];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + params.w_x[i * config.input_size + j] * input[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + params.w_h[i * config.hidden_size + k] * hidden[k];
+ k = k + 1;
+ }
+ new_hidden[i] = tanh(z);
+ i = i + 1;
+ }
+ }
+
+ // inputs: steps x input_size; outputs: steps x hidden_size, row t = h after step t.
+ // The state starts at initial_state and each step reads the previous row.
+ fn forward_sequence(inputs: []const f32, steps: usize, initial_state: []const f32, params: RNNParams, config: RNNCellConfig, outputs: []f32) -> void {
+ const n_in : usize = config.input_size;
+ const n_h : usize = config.hidden_size;
+ var t : usize = 0;
+ while (t < steps) {
+ if (t == 0) {
+ forward_step(inputs[0..n_in], initial_state, params, config, outputs[0..n_h]);
+ } else {
+ forward_step(inputs[t * n_in .. (t + 1) * n_in], outputs[(t - 1) * n_h .. t * n_h], params, config, outputs[t * n_h .. (t + 1) * n_h]);
+ }
+ t = t + 1;
+ }
+ }
+
+ // The first hidden_size entries of `buffer`, zeroed
+ fn init_state(buffer: []f32, hidden_size: u32) -> []f32 {
+ var i : usize = 0;
+ while (i < hidden_size) {
+ buffer[i] = 0.0;
+ i = i + 1;
+ }
+ return buffer[0..hidden_size];
+ }
+
+ // A 1-in, 1-hidden cell with W_x = 1, W_h = 0.5, b = 0: h' = tanh(x + 0.5 h)
+ fn tiny() -> RNNCellConfig {
+ return RNNCellConfig{ .input_size = 1, .hidden_size = 1 };
+ }
+
+ fn tiny_step(x: f32, h: f32) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const input : [1]f32 = [x];
+ const hidden : [1]f32 = [h];
+ var out : [1]f32 = [9.0];
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &out);
+ return out[0];
+ }
+
+ // The sequence (1, 0, 0) from h = 0: h after step t
+ fn tiny_sequence(t: usize) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const inputs : [3]f32 = [1.0, 0.0, 0.0];
+ const h0 : [1]f32 = [0.0];
+ var outputs : [3]f32 = [9.0] ** 3;
+ forward_sequence(&inputs, 3, &h0, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &outputs);
+ return outputs[t];
+ }
+
+ // A 2-in, 2-hidden cell: h' = tanh(W_x x + W_h h + b) with W_x = [[1, -1], [0.5, 0.5]],
+ // W_h = [[0.1, 0], [0, 0.1]], b = (0, -0.5), at x = (1, 2), h = (1, -1)
+ fn wide_step(index: usize) -> f32 {
+ var w_x : [4]f32 = [1.0, -1.0, 0.5, 0.5];
+ var w_h : [4]f32 = [0.1, 0.0, 0.0, 0.1];
+ var b : [2]f32 = [0.0, -0.5];
+ const input : [2]f32 = [1.0, 2.0];
+ const hidden : [2]f32 = [1.0, -1.0];
+ var out : [2]f32 = [9.0] ** 2;
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, RNNCellConfig{ .input_size = 2, .hidden_size = 2 }, &out);
+ return out[index];
}
- // forward_sequence(inputs: [][]f32, initial_state: []f32, params: RNNParams) -> [][]f32
- fn forward_sequence(inputs: [][]f32, initial_state: []f32, params: RNNParams) -> [][]f32 {
- // Basic RNN sequence processing implementation
- let sequence_length = len(inputs);
- let mut output_sequence: [][]f32 = [];
-
- // Initialize initial state
- let mut current_state = RNNState {
- hidden: initial_state,
- cell: zeros(len(initial_state)),
- };
-
- // Process each input in sequence
- for i in 0..sequence_length {
- current_state = forward_step(inputs[i], current_state, params);
- output_sequence = append(output_sequence, current_state.hidden);
+ fn init_len(hidden_size: u32) -> usize {
+ var buffer : [64]f32 = [9.0] ** 64;
+ const h = init_state(&buffer, hidden_size);
+ if (h[hidden_size - 1] != 0.0) {
+ return 0;
}
-
- return output_sequence;
+ return h.len;
}
- // init_state(hidden_size: u32) -> []f32
- fn init_state(hidden_size: u32) -> []f32 {
- // Initialize RNN state with zeros
- return zeros(hidden_size);
+ fn valid_input() -> f32 {
+ return tiny_step(1.0, 0.0);
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // z = (1 - 2 + 0.1, 0.5 + 1 - 0.1 - 0.5) = (-0.9, 0.9): h' = (tanh(-0.9), tanh(0.9))
test forward_step_basic_case
- given input = default_input()
- state = default_state()
- params = default_params()
- when result = forward_step(input, state, params)
- then result != undefined
+ given h0 = wide_step(0)
+ when h1 = wide_step(1)
+ then @abs(h0 + 0.7162979) < 0.000001 and @abs(h1 - 0.7162979) < 0.000001 and @abs(tiny_step(1.0, 0.0) - 0.7615942) < 0.000001
+ // The cell remembers: after the input pulse, h decays through the recurrent weight,
+ // h1 = tanh(1), h2 = tanh(0.5 h1), h3 = tanh(0.5 h2)
test forward_sequence_basic_case
- given inputs = default_sequence_inputs()
- initial_state = default_input()
- params = default_params()
- when result = forward_sequence(inputs, initial_state, params)
- then result != undefined
+ given h1 = tiny_sequence(0)
+ when h3 = tiny_sequence(2)
+ then @abs(h1 - 0.7615942) < 0.000001 and @abs(tiny_sequence(1) - tanh(0.5 * h1)) < 0.000001 and @abs(h3 - tanh(0.5 * tiny_sequence(1))) < 0.000001 and h3 > 0.0 and h3 < tiny_sequence(1)
test init_state_basic_case
given hidden_size = 64
- when result = init_state(hidden_size)
- then result != undefined
- then len(result) == hidden_size
+ when result = init_len(hidden_size)
+ then result == hidden_size
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // input_size >= 1
invariant rnn_cell_constraint_0
- given input = valid_input()
- then true // input_size >= 1
+ assert DEFAULT_INPUT_SIZE >= 1
+ // hidden_size >= 1
invariant rnn_cell_constraint_1
- given input = valid_input()
- then true // hidden_size >= 1
+ assert DEFAULT_HIDDEN_SIZE >= 1
+ // name: rnn_formula -- one tanh, not two: tanh(1) = 0.7615942, tanh(tanh(1)) = 0.6420
invariant rnn_cell_constraint_2
- given input = valid_input()
- then true // name: rnn_formula
+ assert valid_input() > 0.76 and valid_input() < 0.77
+ // "Elman (1990) - Finding Structure in Time": the state carries the past forward
invariant rnn_cell_constraint_3
- given input = valid_input()
- then true // "Elman (1990) - Finding Structure in Time"
+ assert tiny_step(0.0, 1.0) > 0.0
+ // "Goodfellow et al. (2016) - Sequence Modeling with RNNs and LSTMs": tanh bounds the state
invariant rnn_cell_constraint_4
- given input = valid_input()
- then true // "Goodfellow et al. (2016) - Sequence Modeling with RNNs and LSTMs"
\ No newline at end of file
+ assert tiny_step(100.0, 1.0) <= 1.0 and tiny_step(-100.0, -1.0) >= -1.0
diff --git a/apps/website/public/t27/files/specs/ml/rl/ppo_clip_loss.t27 b/apps/website/public/t27/files/specs/ml/rl/ppo_clip_loss.t27
index 637f851d91..72d30bb304 100644
--- a/apps/website/public/t27/files/specs/ml/rl/ppo_clip_loss.t27
+++ b/apps/website/public/t27/files/specs/ml/rl/ppo_clip_loss.t27
@@ -2,6 +2,15 @@
// t27/specs/
// Probability ratio between policies | φ² + 1/φ² = 3 | TRINITY
+// #8980: written with `let x = if ... {}` and `for i in 0..n`, so it never compiled.
+// Three formulas were wrong, and their tests pinned the wrong values:
+// - compute_ratio returned log pi_new - log pi_old; the ratio is exp of that.
+// - clipped_surrogate returned clip(r) A; PPO's term is min(r A, clip(r) A), so
+// r = 1.1 inside the clip range gives 2.2, not 2.4.
+// - policy_loss returned the surrogate itself, a quantity to MAXIMISE; as a loss
+// added to the value and entropy losses it is minus the mean surrogate.
+// Each test now states the value PPO defines (Schulman et al., 2017).
+
module PpoClipLoss;
use base::types;
use math::constants;
@@ -36,68 +45,72 @@ module PpoClipLoss;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // compute_ratio(new_log_prob: f32, old_log_prob: f32) → f32
+ // r = pi_new / pi_old = exp(log pi_new - log pi_old)
fn compute_ratio(new_log_prob: f32, old_log_prob: f32) -> f32 {
- // Compute probability ratio between new and old policies
- return new_log_prob - old_log_prob;
+ return exp(new_log_prob - old_log_prob);
}
- // clipped_surrogate(ratio: f32, advantage: f32, clip_epsilon: f32) → f32
+ // min(r A, clip(r, 1 - eps, 1 + eps) A)
fn clipped_surrogate(ratio: f32, advantage: f32, clip_epsilon: f32) -> f32 {
- // Apply clipping to the surrogate objective
- let clipped_ratio = if ratio > 1.0 + clip_epsilon {
- 1.0 + clip_epsilon
- } else if ratio < 1.0 - clip_epsilon {
- 1.0 - clip_epsilon
- } else {
- ratio
- };
- return clipped_ratio * advantage;
+ const clipped_ratio = @min(@max(ratio, 1.0 - clip_epsilon), 1.0 + clip_epsilon);
+ return @min(ratio * advantage, clipped_ratio * advantage);
}
- // policy_loss(ratios: []f32, advantages: []f32, clip_epsilon: f32) → f32
- fn policy_loss(ratios: []f32, advantages: []f32, clip_epsilon: f32) -> f32 {
- // Compute policy loss using clipped surrogate objective
- let total_loss = 0.0;
- for i in 0..ratios.len {
- let ratio = ratios[i];
- let advantage = advantages[i];
- let clipped_loss = clipped_surrogate(ratio, advantage, clip_epsilon);
- total_loss += clipped_loss;
+ // -mean_t min(r_t A_t, clip(r_t) A_t): the loss gradient descent minimises
+ fn policy_loss(ratios: []const f32, advantages: []const f32, clip_epsilon: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < ratios.len) {
+ total = total + clipped_surrogate(ratios[i], advantages[i], clip_epsilon);
+ i = i + 1;
}
- return total_loss / f32(ratios.len);
+ return -total / @as(f32, @floatFromInt(ratios.len));
}
- // entropy_loss(entropies: []f32, entropy_coef: f32) → f32
- fn entropy_loss(entropies: []f32, entropy_coef: f32) -> f32 {
- // Compute entropy loss to encourage exploration
- let total_entropy = 0.0;
- for entropy in entropies {
- total_entropy += entropy;
+ // -coef * mean entropy: lower loss for a more exploratory policy
+ fn entropy_loss(entropies: []const f32, entropy_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < entropies.len) {
+ total = total + entropies[i];
+ i = i + 1;
}
- return -entropy_coef * (total_entropy / f32(entropies.len));
+ return -entropy_coef * (total / @as(f32, @floatFromInt(entropies.len)));
}
- // value_loss(predicted_values: []f32, target_values: []f32, old_values: []f32, clip_coef: f32) → f32
- fn value_loss(predicted_values: []f32, target_values: []f32, old_values: []f32, clip_coef: f32) -> f32 {
- // Compute value loss using clipped value function
- let total_loss = 0.0;
- for i in 0..predicted_values.len {
- let predicted = predicted_values[i];
- let target = target_values[i];
- let old = old_values[i];
- let value_diff = predicted - old;
- let clipped_value_diff = if value_diff > clip_coef {
- clip_coef
- } else if value_diff < -clip_coef {
- -clip_coef
- } else {
- value_diff
- };
- let value_loss = 0.5 * (target - (old + clipped_value_diff)) * (target - (old + clipped_value_diff));
- total_loss += value_loss;
+ // mean 0.5 (target - (old + clip(pred - old, -c, c)))^2
+ fn value_loss(predicted_values: []const f32, target_values: []const f32, old_values: []const f32, clip_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predicted_values.len) {
+ const diff = @min(@max(predicted_values[i] - old_values[i], -clip_coef), clip_coef);
+ const err = target_values[i] - (old_values[i] + diff);
+ total = total + 0.5 * err * err;
+ i = i + 1;
}
- return total_loss / f32(predicted_values.len);
+ return total / @as(f32, @floatFromInt(predicted_values.len));
+ }
+
+ fn policy_loss_of(r0: f32, r1: f32, r2: f32) -> f32 {
+ const ratios : [3]f32 = [r0, r1, r2];
+ const advantages : [3]f32 = [2.0, 1.5, 3.0];
+ return policy_loss(&ratios, &advantages, DEFAULT_CLIP_EPSILON);
+ }
+
+ fn entropy_loss_of() -> f32 {
+ const entropies : [3]f32 = [0.5, 0.6, 0.4];
+ return entropy_loss(&entropies, DEFAULT_ENTROPY_COEF);
+ }
+
+ fn value_loss_of(p0: f32) -> f32 {
+ const predicted : [3]f32 = [p0, 2.0, 1.5];
+ const target : [3]f32 = [1.1, 2.1, 1.6];
+ const old : [3]f32 = [0.9, 1.9, 1.4];
+ return value_loss(&predicted, &target, &old, 0.2);
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_CLIP_EPSILON;
}
// total_loss(policy_loss_value: f32, value_loss_value: f32, entropy_loss_value: f32, value_loss_coef: f32) → f32
@@ -115,7 +128,7 @@ module PpoClipLoss;
new_log_prob = 0.5
old_log_prob = 0.3
when result = compute_ratio(new_log_prob, old_log_prob)
- then result == 0.2
+ then @abs(result - 1.2214028) < 0.000001 // exp(0.2); the old `== 0.2` was the log ratio
test clipped_surrogate_basic_case
given
@@ -123,7 +136,7 @@ module PpoClipLoss;
advantage = 2.0
clip_epsilon = 0.2
when result = clipped_surrogate(ratio, advantage, clip_epsilon)
- then result == 1.2 * 2.0
+ then @abs(result - 1.1 * 2.0) < 0.000001 // inside [0.8, 1.2] the clip does nothing
test clipped_surrogate_clipping_case
given
@@ -131,31 +144,23 @@ module PpoClipLoss;
advantage = 2.0
clip_epsilon = 0.2
when result = clipped_surrogate(ratio, advantage, clip_epsilon)
- then result == 1.2 * 2.0
+ then @abs(result - 1.2 * 2.0) < 0.000001 and clipped_surrogate(0.5, -1.0, 0.2) == -0.8
+ // min(r A, clip(r) A) = (2.0, 1.65, 2.7): the loss is -mean = -2.1166667. The old `> 0` was
+ // the surrogate's sign; a ratio pushed past the clip earns nothing more.
test policy_loss_basic_case
- given
- ratios = [1.0, 1.1, 0.9]
- advantages = [2.0, 1.5, 3.0]
- clip_epsilon = 0.2
- when result = policy_loss(ratios, advantages, clip_epsilon)
- then result > 0.0
+ given result = policy_loss_of(1.0, 1.1, 0.9)
+ when pushed = policy_loss_of(1.0, 1.1, 3.0)
+ then result < 0.0 and @abs(result + 2.1166667) < 0.000001 and @abs(pushed - policy_loss_of(1.0, 1.1, 1.2)) < 0.000001
test entropy_loss_basic_case
- given
- entropies = [0.5, 0.6, 0.4]
- entropy_coef = 0.01
- when result = entropy_loss(entropies, entropy_coef)
- then result < 0.0
+ given result = entropy_loss_of()
+ then result < 0.0 and @abs(result + 0.005) < 0.0000001
test value_loss_basic_case
- given
- predicted_values = [1.0, 2.0, 1.5]
- target_values = [1.1, 2.1, 1.6]
- old_values = [0.9, 1.9, 1.4]
- clip_coef = 0.2
- when result = value_loss(predicted_values, target_values, old_values, clip_coef)
- then result > 0.0
+ given result = value_loss_of(1.0)
+ when clipped = value_loss_of(5.0)
+ then result > 0.0 and @abs(result - 0.005) < 0.0000001 and @abs(clipped - value_loss_of(1.1)) < 0.0000001
test total_loss_basic_case
given
@@ -164,29 +169,29 @@ module PpoClipLoss;
entropy_loss_value = -0.01
value_loss_coef = 0.5
when result = total_loss(policy_loss_value, value_loss_value, entropy_loss_value, value_loss_coef)
- then result == 1.0 + 0.5 * 0.5 + (-0.01)
+ then @abs(result - (1.0 + 0.5 * 0.5 + (-0.01))) < 0.0000001
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // 0 < clip_epsilon <= 1 (typically 0.1-0.3)
invariant ppo_clip_loss_constraint_0
- given input = valid_input()
- then true // 0 < clip_epsilon <= 1 (typically 0.1-0.3)
+ assert valid_input() > 0.0 and valid_input() <= 1.0
+ // entropy_coef >= 0
invariant ppo_clip_loss_constraint_1
- given input = valid_input()
- then true // entropy_coef >= 0
+ assert DEFAULT_ENTROPY_COEF >= 0.0
+ // name: ppo_objective -- for A > 0 the surrogate stops growing at r = 1 + eps
invariant ppo_clip_loss_constraint_2
- given input = valid_input()
- then true // name: ppo_objective
+ assert clipped_surrogate(5.0, 1.0, 0.2) == 1.2
+ // name: probability_ratio -- equal log-probabilities give r = 1
invariant ppo_clip_loss_constraint_3
- given input = valid_input()
- then true // name: probability_ratio
+ assert compute_ratio(0.3, 0.3) == 1.0
+ // "Schulman et al. (2017) - Proximal Policy Optimization Algorithms, arXiv:1707.06347":
+ // for A < 0 the surrogate stops falling at r = 1 - eps
invariant ppo_clip_loss_constraint_4
- given input = valid_input()
- then true // "Schulman et al. (2017) - Proximal Policy Optimization Algorithms, arXiv:1707.06347"
-
+ assert clipped_surrogate(0.1, -1.0, 0.2) == -0.8
diff --git a/apps/website/public/t27/files/specs/ml/transformer/encoder_block.t27 b/apps/website/public/t27/files/specs/ml/transformer/encoder_block.t27
index bedc26f64c..1ced0caac4 100644
--- a/apps/website/public/t27/files/specs/ml/transformer/encoder_block.t27
+++ b/apps/website/public/t27/files/specs/ml/transformer/encoder_block.t27
@@ -2,6 +2,14 @@
// t27/specs/
// Standard Transformer encoder block with residual connections | φ² + 1/φ² = 3 | TRINITY
+// #8980: attention and the FFN returned their input ("in real implementation
+// ..."), forward skipped both residuals, and the tests asked only for a non-empty
+// output from undeclared `default_input()`. The block now computes the Transformer
+// encoder layer over a seq x d_model input. Attention is multi-head scaled
+// dot-product, softmax(q k^T / sqrt(d_head)) v per head, with identity projections.
+// Then come the position-wise FFN and two residual + layer-norm steps: post-norm
+// LN(x + f(x)) by default, pre-norm x + f(LN(x)) when use_pre_norm is set.
+
module EncoderBlock;
use base::types;
use math::constants;
@@ -27,105 +35,337 @@ module EncoderBlock;
use_pre_norm : bool,
};
- pub const AttentionOutput = struct {
- output : []f32,
- attn_weights : [][]f32,
- };
-
- pub const FFNOutput = struct {
- output : []f32,
- };
-
- pub const BlockOutput = struct {
- output : []f32,
+ pub const FFNWeights = struct {
+ w1 : []const f32,
+ b1 : []const f32,
+ w2 : []const f32,
+ b2 : []const f32,
};
// ═══════════════════════════════════════════════════════════
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // multi_head_attention(query: []f32, key: []f32, value: []f32, n_heads: u32, d_model: u32) → AttentionOutput
- fn multi_head_attention(query: []f32, key: []f32, value: []f32, n_heads: u32, d_model: u32) -> AttentionOutput {
- // Basic multi-head attention implementation
- // For now, return a simple identity transformation
- let output = query; // In real implementation, this would be the attention output
- let attn_weights = []; // In real implementation, this would contain attention weights
-
- return AttentionOutput {
- output: output,
- attn_weights: attn_weights,
- };
+ // q, k, v and out are seq x d_model; attn_weights is n_heads x seq x seq. Head h uses
+ // columns h * d_head .. (h + 1) * d_head.
+ fn multi_head_attention(query: []const f32, key: []const f32, value: []const f32, seq: usize, n_heads: u32, d_model: u32, out: []f32, attn_weights: []f32) -> void {
+ const d_head : usize = d_model / n_heads;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d_head)));
+ var h : usize = 0;
+ while (h < n_heads) {
+ var i : usize = 0;
+ while (i < seq) {
+ const row = attn_weights[(h * seq + i) * seq .. (h * seq + i + 1) * seq];
+ var max_s : f32 = -1.0e30;
+ var j : usize = 0;
+ while (j < seq) {
+ var dot : f32 = 0.0;
+ var c : usize = 0;
+ while (c < d_head) {
+ dot = dot + query[i * d_model + h * d_head + c] * key[j * d_model + h * d_head + c];
+ c = c + 1;
+ }
+ row[j] = dot * scale;
+ max_s = @max(max_s, row[j]);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ var c : usize = 0;
+ while (c < d_head) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ acc = acc + row[j] / sum * value[j * d_model + h * d_head + c];
+ j = j + 1;
+ }
+ out[i * d_model + h * d_head + c] = acc;
+ c = c + 1;
+ }
+ j = 0;
+ while (j < seq) {
+ row[j] = row[j] / sum;
+ j = j + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ // out = W2 relu(W1 x + b1) + b2 for one row; hidden is d_ff floats of scratch.
+ // Dropout is the identity at inference, which is what this block computes.
+ fn feed_forward(input: []const f32, weights: FFNWeights, d_model: u32, d_ff: u32, hidden: []f32, out: []f32) -> void {
+ var f : usize = 0;
+ while (f < d_ff) {
+ var z : f32 = weights.b1[f];
+ var j : usize = 0;
+ while (j < d_model) {
+ z = z + weights.w1[f * d_model + j] * input[j];
+ j = j + 1;
+ }
+ hidden[f] = @max(z, 0.0);
+ f = f + 1;
+ }
+ var o : usize = 0;
+ while (o < d_model) {
+ var y : f32 = weights.b2[o];
+ f = 0;
+ while (f < d_ff) {
+ y = y + weights.w2[o * d_ff + f] * hidden[f];
+ f = f + 1;
+ }
+ out[o] = y;
+ o = o + 1;
+ }
+ }
+
+ // (x - mean) / sqrt(var + 1e-5), no learned scale or shift
+ fn layer_norm(x: []const f32, out: []f32) -> void {
+ var mean : f32 = 0.0;
+ var k : usize = 0;
+ while (k < x.len) {
+ mean = mean + x[k];
+ k = k + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(x.len));
+ var variance : f32 = 0.0;
+ k = 0;
+ while (k < x.len) {
+ variance = variance + (x[k] - mean) * (x[k] - mean);
+ k = k + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(x.len));
+ k = 0;
+ while (k < x.len) {
+ out[k] = (x[k] - mean) / sqrt(variance + 0.00001);
+ k = k + 1;
+ }
+ }
+
+ // input and out are seq x d_model. scratch needs 3 seq d_model + n_heads seq^2 + d_ff + d_model floats.
+ fn forward(input: []const f32, seq: usize, ffn: FFNWeights, config: EncoderBlockConfig, out: []f32, scratch: []f32) -> void {
+ const n : usize = seq * config.d_model;
+ const d : usize = config.d_model;
+ const normed = scratch[0..n];
+ const attn = scratch[n .. 2 * n];
+ const mid = scratch[2 * n .. 3 * n];
+ const weights = scratch[3 * n .. 3 * n + config.n_heads * seq * seq];
+ const rest = 3 * n + config.n_heads * seq * seq;
+ const hidden = scratch[rest .. rest + config.d_ff];
+ const row = scratch[rest + config.d_ff .. rest + config.d_ff + d];
+ var t : usize = 0;
+ if (config.use_pre_norm) {
+ while (t < seq) {
+ layer_norm(input[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ multi_head_attention(normed, normed, normed, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ mid[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(mid[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ feed_forward(normed[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ out[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ t = t + 1;
+ }
+ return;
+ }
+ multi_head_attention(input, input, input, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ normed[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(normed[t * d .. (t + 1) * d], mid[t * d .. (t + 1) * d]);
+ feed_forward(mid[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ normed[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ layer_norm(normed[t * d .. (t + 1) * d], out[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads, d_ff 8
+ fn cfg(pre_norm: bool) -> EncoderBlockConfig {
+ return EncoderBlockConfig{ .d_model = 4, .n_heads = 2, .d_ff = 8, .dropout = DEFAULT_DROPOUT, .use_pre_norm = pre_norm };
}
- // feed_forward(input: []f32, d_model: u32, d_ff: u32, dropout: f32) → FFNOutput
- fn feed_forward(input: []f32, d_model: u32, d_ff: u32, dropout: f32) -> FFNOutput {
- // Basic feed-forward network implementation
- // For now, return a simple identity transformation
- // In real implementation, this would be: Linear → ReLU → Linear → Dropout
-
- return FFNOutput {
- output: input, // In real implementation, this would be the FFN output
- };
+ fn fixture_input(x: []f32) -> void {
+ const v : [12]f32 = [1.0, 0.0, 0.5, -1.0, 0.0, 1.0, -0.5, 2.0, 1.0, 1.0, 0.0, 0.0];
+ var k : usize = 0;
+ while (k < 12) {
+ x[k] = v[k];
+ k = k + 1;
+ }
}
- // forward(input: []f32, config: EncoderBlockConfig) → BlockOutput
- fn forward(input: []f32, config: EncoderBlockConfig) -> BlockOutput {
- // Basic encoder block forward pass implementation
- // Standard transformer block: x → LayerNorm → Attention → Add → LayerNorm → FFN → Add
-
- // Apply multi-head attention with residual connection
- let attn_output = multi_head_attention(input, input, input, config.n_heads, config.d_model);
- let attn_with_residual = attn_output.output; // In real implementation: add input here
-
- // Apply feed-forward network with residual connection
- let ffn_output = feed_forward(attn_with_residual, config.d_model, config.d_ff, config.dropout);
- let final_output = ffn_output.output; // In real implementation: add attn_with_residual here
-
- return BlockOutput {
- output: final_output,
- };
+ // which: 0 = out[index], 1 = attn_weights[index] of self-attention on the fixture
+ fn attention_at(which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var out : [12]f32 = [9.0] ** 12;
+ var weights : [18]f32 = [9.0] ** 18;
+ multi_head_attention(&x, &x, &x, 3, 2, 4, &out, &weights);
+ if (which == 0) {
+ return out[index];
+ }
+ return weights[index];
+ }
+
+ // Two keys equal: attention splits evenly and returns the mean of the values
+ fn tie_attention(index: usize) -> f32 {
+ const q : [2]f32 = [1.0, 1.0];
+ const k : [2]f32 = [2.0, 2.0];
+ const v : [2]f32 = [4.0, 10.0];
+ var out : [2]f32 = [0.0] ** 2;
+ var weights : [4]f32 = [0.0] ** 4;
+ multi_head_attention(&q, &k, &v, 2, 1, 1, &out, &weights);
+ return out[index];
+ }
+
+ fn ffn_fill(w1: []f32, b1: []f32, w2: []f32, b2: []f32) -> void {
+ var k : usize = 0;
+ while (k < 32) {
+ w1[k] = @as(f32, @floatFromInt((k * 5 + 1) % 9)) / 8.0 - 0.5;
+ w2[k] = @as(f32, @floatFromInt((k * 7 + 2) % 9)) / 8.0 - 0.5;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 8) {
+ b1[k] = 0.1;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 4) {
+ b2[k] = -0.1;
+ k = k + 1;
+ }
+ }
+
+ // which: 0 = out[index]; 1 = mean of row `index`; 2 = variance of row `index`
+ fn block_at(pre_norm: bool, which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ var out : [12]f32 = [9.0] ** 12;
+ var scratch : [3 * 12 + 18 + 8 + 4]f32 = undefined;
+ forward(&x, 3, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, cfg(pre_norm), &out, &scratch);
+ if (which == 0) {
+ return out[index];
+ }
+ var mean : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ mean = mean + out[index * 4 + c] / 4.0;
+ c = c + 1;
+ }
+ if (which == 1) {
+ return mean;
+ }
+ var variance : f32 = 0.0;
+ c = 0;
+ while (c < 4) {
+ variance = variance + (out[index * 4 + c] - mean) * (out[index * 4 + c] - mean) / 4.0;
+ c = c + 1;
+ }
+ return variance;
+ }
+
+ // FFN on row (1, 0, 0.5, -1) of the fixture weights; out[index]
+ fn ffn_at(index: usize) -> f32 {
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ const x : [4]f32 = [1.0, 0.0, 0.5, -1.0];
+ var hidden : [8]f32 = undefined;
+ var out : [4]f32 = [9.0] ** 4;
+ feed_forward(&x, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, 4, 8, &hidden, &out);
+ return out[index];
+ }
+
+ fn valid_input() -> u32 {
+ return DEFAULT_D_MODEL / DEFAULT_N_HEADS;
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // Every attention row is a distribution; equal keys give the mean of the values
test multi_head_attention_basic_case
- given query = default_input(), key = default_input(), value = default_input(), n_heads = DEFAULT_N_HEADS, d_model = DEFAULT_D_MODEL
- when result = multi_head_attention(query, key, value, n_heads, d_model)
- then result.output.len > 0 and result.attn_weights.len > 0
+ given row_sum = attention_at(1, 0) + attention_at(1, 1) + attention_at(1, 2)
+ when tied = tie_attention(0)
+ then @abs(row_sum - 1.0) < 0.000001 and tied == 7.0 and tie_attention(1) == 7.0 and attention_at(0, 0) != 9.0
+ // Heads are independent: head 1's weights (rows 9..17) differ from head 0's
+ test multi_head_attention_heads_differ
+ given w0 = attention_at(1, 1)
+ when w1 = attention_at(1, 10)
+ then w0 != w1 and w0 > 0.0 and w1 > 0.0
+
+ // W2 relu(W1 x + b1) + b2 on the fixture row, worked by hand: (0.4125, ..., -0.178125)
test feed_forward_basic_case
- given input = default_input(), d_model = DEFAULT_D_MODEL, d_ff = DEFAULT_D_FF, dropout = DEFAULT_DROPOUT
- when result = feed_forward(input, d_model, d_ff, dropout)
- then result.output.len > 0
+ given y0 = ffn_at(0)
+ when y3 = ffn_at(3)
+ then @abs(y0 - 0.4125) < 0.000001 and @abs(y3 + 0.178125) < 0.000001 and @abs(ffn_at(2) + 0.928125) < 0.000001
+ // Post-norm: every output row has mean 0 and variance 1
test forward_basic_case
- given input = default_input(), config = EncoderBlockConfig { d_model = DEFAULT_D_MODEL, n_heads = DEFAULT_N_HEADS, d_ff = DEFAULT_D_FF, dropout = DEFAULT_DROPOUT, use_pre_norm = true }
- when result = forward(input, config)
- then result.output.len > 0
+ given mean1 = block_at(false, 1, 1)
+ when var1 = block_at(false, 2, 1)
+ then @abs(mean1) < 0.00001 and @abs(var1 - 1.0) < 0.001 and @abs(block_at(false, 1, 2)) < 0.00001
+
+ // Pre-norm keeps the residual stream un-normalised, so it differs from post-norm
+ test forward_pre_norm_differs
+ given pre = block_at(true, 0, 0)
+ when post = block_at(false, 0, 0)
+ then pre != post and pre == pre
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
+ // d_model % n_heads == 0
invariant encoder_block_constraint_0
- given input = valid_input()
- then true // d_model % n_heads == 0
+ assert DEFAULT_D_MODEL % DEFAULT_N_HEADS == 0 and cfg(false).d_model % cfg(false).n_heads == 0
+ // d_ff >= d_model
invariant encoder_block_constraint_1
- given input = valid_input()
- then true // d_ff >= d_model
+ assert DEFAULT_D_FF >= DEFAULT_D_MODEL and cfg(false).d_ff >= cfg(false).d_model
+ // name: encoder_block -- each head gets d_model / n_heads columns
invariant encoder_block_constraint_2
- given input = valid_input()
- then true // name: encoder_block
+ assert valid_input() == 64
+ // "Vaswani et al. (2017) - Attention Is All You Need": dropout is a probability
invariant encoder_block_constraint_3
- given input = valid_input()
- then true // "Vaswani et al. (2017) - Attention Is All You Need"
+ assert DEFAULT_DROPOUT >= 0.0 and DEFAULT_DROPOUT < 1.0
+ // "Baevski et al. (2020) - Revisiting the Self-Attention": both norm placements are offered
invariant encoder_block_constraint_4
- given input = valid_input()
- then true // "Baevski et al. (2020) - Revisiting the Self-Attention"
-
+ assert cfg(true).use_pre_norm and !cfg(false).use_pre_norm
diff --git a/apps/website/public/t27/files/specs/ml/transformer/positional_encoding.t27 b/apps/website/public/t27/files/specs/ml/transformer/positional_encoding.t27
index b1ca8a4650..11ed097e9a 100644
--- a/apps/website/public/t27/files/specs/ml/transformer/positional_encoding.t27
+++ b/apps/website/public/t27/files/specs/ml/transformer/positional_encoding.t27
@@ -2,6 +2,13 @@
// t27/specs/
// Uses sin/cos at different frequencies for each dimension | φ² + 1/φ² = 3 | TRINITY
+// #8980: forward read an undeclared `d_model`, called `math::pow` / `math::sin` /
+// `math::cos`, computed each encoding into a local and dropped it, and grew the
+// frequency with the dimension where Vaswani et al. (2017) shrink it. It now
+// writes the sinusoidal encoding
+// PE(pos, 2i) = sin(pos / 10000^(2i / d)), PE(pos, 2i + 1) = cos(pos / 10000^(2i / d))
+// for each position into a positions x d_model buffer.
+
module PositionalEnc;
use base::types;
use math::constants;
@@ -25,39 +32,57 @@ module PositionalEnc;
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
- // forward(positions: []const u32) → void
- fn forward(positions: []const u32) -> void {
- // Compute positional encodings using sin/cos at different frequencies
- // Each position gets a vector where even indices use sin and odd use cos
- for (i in positions) {
- // For each position, compute encodings for all dimensions
- // Frequency increases by powers of 2: 1/10000, 1/10000^2, 1/10000^3, etc.
- let freq_base = 1.0 / 10000.0;
- for (dim in 0..d_model) {
- let freq = freq_base * math::pow(2.0, cast(f64) dim / 2.0);
+ // 1 / 10000^(2i / d) = exp(-(2i / d) ln 10000)
+ fn inv_freq(pair: u32, d_model: u32) -> f32 {
+ return exp(-(@as(f32, @floatFromInt(2 * pair)) / @as(f32, @floatFromInt(d_model))) * log(@as(f32, 10000.0)));
+ }
+
+ fn forward(positions: []const u32, config: PosEncConfig, out: []f32) -> void {
+ var p : usize = 0;
+ while (p < positions.len) {
+ const pos : f32 = @as(f32, @floatFromInt(positions[p]));
+ var dim : u32 = 0;
+ while (dim < config.d_model) {
+ const angle = pos * inv_freq(dim / 2, config.d_model);
if (dim % 2 == 0) {
- // Even dimensions: sin(pos / freq)
- let encoding = math::sin(cast(f64) i * freq);
+ out[p * config.d_model + dim] = sin(angle);
} else {
- // Odd dimensions: cos(pos / freq)
- let encoding = math::cos(cast(f64) i * freq);
+ out[p * config.d_model + dim] = cos(angle);
}
- // Store the encoding (implementation would typically store in an output array)
+ dim = dim + 1;
}
+ p = p + 1;
}
}
+ // PE(pos, dim) for d_model 8
+ fn pe(pos: u32, dim: usize) -> f32 {
+ const positions : [1]u32 = [pos];
+ var out : [8]f32 = [9.0] ** 8;
+ forward(&positions, PosEncConfig{ .d_model = 8, .max_len = DEFAULT_MAX_LEN }, &out);
+ return out[dim];
+ }
+
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
+ // pos 1, d 8: dims 0/1 use frequency 1, dims 2/3 use 1/10 (10000^(2/8) = 10)
test forward_basic_case
- given input = default_input()
- when result = forward(input)
- then result != undefined
+ given s0 = pe(1, 0)
+ when s2 = pe(1, 2)
+ then @abs(s0 - 0.8414710) < 0.000001 and @abs(pe(1, 1) - 0.5403023) < 0.000001 and @abs(s2 - 0.0998334) < 0.000001 and @abs(pe(1, 3) - 0.9950042) < 0.000001
+ // Position 0 encodes as (0, 1, 0, 1, ...)
test forward_simple
- given input = [0u32]
- when result = forward(input)
- then result != undefined
+ given s = pe(0, 6)
+ when c = pe(0, 7)
+ then s == 0.0 and c == 1.0
+
+ // Each sin/cos pair lies on the unit circle
+ test forward_pairs_are_unit_vectors
+ given p = pe(37, 2) * pe(37, 2) + pe(37, 3) * pe(37, 3)
+ then @abs(p - 1.0) < 0.000001
+ invariant positional_encoding_max_len_is_positive
+ assert DEFAULT_MAX_LEN > 0
diff --git a/apps/website/public/widgets/gallery.t27 b/apps/website/public/widgets/gallery.t27
index 6285d9a4d1..17117ac2a9 100644
--- a/apps/website/public/widgets/gallery.t27
+++ b/apps/website/public/widgets/gallery.t27
@@ -78,15 +78,15 @@ pub const TOOL_NOSCRIPT : str = "This widget runs in your browser and needs Java
; Parallel arrays, WIDGET_COUNT long. WIDGET_TITLES is empty wherever the page carries an
; og:title, which is every kind but tab: the page is the one place the words live. A tool's
; WIDGET_HOOKS entry is empty too: its hook is the page's t27:hook, written from its own spec.
-pub const WIDGET_COUNT : u8 = 252;
-pub const WIDGET_IDS : [252]str = ["x7-board", "tri-x7-fasm", "tri-x7-mutate", "perf2-loop", "tri-fpga-rgmii", "tri-fpga-jtag", "tri-fpga-pins", "devkit-flow", "play-verilog", "bit-diff", "build-receipt", "cost-curves", "gf16-calc", "idcode", "lut-explain", "pin-map", "placement-map", "resource-bars", "slack-waterfall", "test-waves", "synth-smells", "bit-autopsy", "fasm-skyline", "vcd-wrapped", "fits-on", "stat-card", "lut-depth", "fsm-sketch", "lut-treemap", "t27-cell", "t27-vs-nvfp4", "uart-bucket", "tri-fpga-specs", "tri-fpga-tools", "tri-fpga-wire", "tri-fpga-batch", "tri-fpga-status", "tri-fpga-tmpcheck", "tri-fpga-keycheck", "tri-fpga-selftest", "tri-x7-refs", "tri-x7-audit", "t27c-clock-domain", "t27c-clocks", "t27c-wukong-v1", "t27c-cts", "t27c-mmcm", "t27c-mmcm-mutate", "t27c-timing", "t27c-reset-sync", "t27c-reset-sync-gen", "t27c-mtbf", "t27c-clock-domain-gen", "t27c-fifo", "t27c-fifo-mutate", "t27c-fifo-flags", "t27c-emitter-xdc", "t27c-fpga-rxcap", "t27-tri-competitors", "t27-tri-discard", "tri-fpga-txhold", "tri-fpga-rxcap", "tri-fpga-rxer", "tri-fpga-steps", "tri-fpga-ioclients", "tri-fpga-claim", "tri-fpga-release", "tri-fpga-next", "t27c-memory", "t27c-spi-tb", "t27c-check-uart", "t27c-check-spi", "t27c-check-apb", "t27c-check-axi4", "t27c-check-bridge", "t27c-check-memory", "t27c-verilog-uart", "t27c-verilog-spi", "t27c-verilog-apb", "t27c-verilog-axi4", "t27c-verilog-bridge", "t27c-verilog-top", "t27c-verilog-uart-tb", "t27c-verilog-apb-tb", "t27c-verilog-axi-tb", "t27c-verilog-mem-tb", "t27c-verilog-integration-tb", "t27c-verilog-top-tb", "t27c-verilog-spi-tb", "t27c-hir-uart", "t27c-hir-apb", "t27c-hir-axi4", "t27c-eth-crc", "t27c-rgmii", "jtag-verdict", "line-bounce", "eye-isi", "ecc-secded", "carry-chain", "prefix-adder", "csa-tree", "shift-add", "booth-radix4", "dsp-fit", "q-format", "round-sat", "sqnr", "cordic", "cordic-vector", "sine-table", "alias-fold", "nco", "dft-bin", "fir-response", "fir-design", "fir-cost", "decimator", "cic-growth", "polyphase", "fft-butterfly", "bit-reverse", "fft-scaling", "barker-corr", "fir-fit", "fir-on-board", "play-hello-world", "play-rust", "play-js", "t27c-hello-world", "t27-tri-misread", "test-touch", "t27c-stages", "t27c-e8m0", "play-e8m0", "t27c-ocp-mx", "play-gfternary", "t27c-gfternary", "play-golden-sieve", "t27c-golden-sieve", "play-tnf17", "t27c-tnf17", "t27c-activation-quantizer", "t27c-bitnet-majority", "t27c-bitnet-neuron-nchunk", "t27c-comb-ternary-dot", "t27c-stream-ternary-mac", "t27c-bitnet-mlp", "t27c-gft-smul", "t27c-gft-sadd", "t27c-gft-signed-mac", "t27c-gft-relu", "t27c-gft-exp2", "t27c-gft-argmax4", "t27c-gft-nll", "t27c-gft-sgd-step", "t27c-gft-xornet", "basics-what-a-spec-is", "basics-the-module-line", "basics-comments-and-prose", "basics-constants", "basics-whole-numbers", "basics-true-false-and-text", "basics-arrays", "basics-trits", "basics-expressions", "basics-a-test-block", "basics-many-tests", "basics-functions", "basics-local-names", "basics-if-and-else", "basics-loops", "basics-pub-or-private", "basics-structs", "basics-enums", "basics-use-other-modules", "basics-invariants", "basics-gen-ts", "basics-seven-backends", "basics-stages-of-t27c", "basics-tri-commands", "basics-reading-errors", "basics-a-small-program", "basics-where-next", "golden-split", "golden-phi-ratio", "golden-lucas", "golden-split-optimality", "golden-radix-economy", "golden-phi-in-gf16", "golden-gf4", "golden-gf6", "golden-gf8", "golden-gf10", "golden-gf12", "golden-gf14", "golden-gf16", "golden-dot-two", "golden-gf24", "golden-gf32", "golden-gf48", "golden-gf64", "golden-gf96", "golden-gf128", "golden-gf256", "golden-gf512", "golden-gf1024", "golden-gft8", "golden-gft16", "golden-gft32", "golden-decode", "t27-tri-vectors", "t27-tri-census", "t27-tri-seals", "t27-tri-prose", "t27-tri-lean", "tri-selftest", "play-terminal-gates", "tc-why-three", "tc-balanced-count", "tc-flip-and-cut", "tc-kleene-gates", "tc-consensus-any", "tc-sum-and-carry", "tc-half-adder", "tc-full-adder", "tc-ripple-carry", "tc-times-a-trit", "tc-long-multiply", "tc-ternary-mac", "tc-two-bits", "tc-five-in-a-byte", "tc-trit-word", "tc-instruction-word", "tc-opcode-map", "tc-immediate", "tc-ternary-alu", "tc-three-way-jump", "tc-run-a-program", "tc-compare", "tc-thirds", "tc-sort-three", "tc-one-neuron", "tc-detector", "tc-tiny-layer", "wars", "tri-game-tick", "tri-game-vault", "perf2-knockouts", "race-the-bee", "gatle", "tri-blog-list", "tri-game-selftest"];
-pub const WIDGET_KINDS : [252]str = ["cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "player", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "player", "player", "player", "cast", "cast", "tool", "tool", "cast", "player", "cast", "player", "cast", "player", "cast", "player", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tab", "cast", "cast", "cast", "tool", "tool", "cast", "cast"];
-pub const WIDGET_CATEGORIES : [252]str = ["fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "game", "game", "game", "game", "game", "game", "game", "game"];
-pub const WIDGET_PAGES : [252]str = ["term/x7-board/", "term/tri-x7-fasm/", "term/tri-x7-mutate/", "term/perf2-loop/", "term/tri-fpga-rgmii/", "term/tri-fpga-jtag/", "term/tri-fpga-pins/", "term/devkit-flow/", "play/hello-world/verilog/", "widgets/bit-diff/", "widgets/build-receipt/", "widgets/cost-curves/", "widgets/gf16-calc/", "widgets/idcode/", "widgets/lut-explain/", "widgets/pin-map/", "widgets/placement-map/", "widgets/resource-bars/", "widgets/slack-waterfall/", "widgets/test-waves/", "widgets/synth-smells/", "widgets/bit-autopsy/", "widgets/fasm-skyline/", "widgets/vcd-wrapped/", "widgets/fits-on/", "widgets/stat-card/", "widgets/lut-depth/", "widgets/fsm-sketch/", "widgets/lut-treemap/", "widgets/t27-cell/", "widgets/t27-vs-nvfp4/", "widgets/uart-bucket/", "term/tri-fpga-specs/", "term/tri-fpga-tools/", "term/tri-fpga-wire/", "term/tri-fpga-batch/", "term/tri-fpga-status/", "term/tri-fpga-tmpcheck/", "term/tri-fpga-keycheck/", "term/tri-fpga-selftest/", "term/tri-x7-refs/", "term/tri-x7-audit/", "term/t27c-clock-domain/", "term/t27c-clocks/", "term/t27c-wukong-v1/", "term/t27c-cts/", "term/t27c-mmcm/", "term/t27c-mmcm-mutate/", "term/t27c-timing/", "term/t27c-reset-sync/", "term/t27c-reset-sync-gen/", "term/t27c-mtbf/", "term/t27c-clock-domain-gen/", "term/t27c-fifo/", "term/t27c-fifo-mutate/", "term/t27c-fifo-flags/", "term/t27c-emitter-xdc/", "term/t27c-fpga-rxcap/", "term/t27-tri-competitors/", "term/t27-tri-discard/", "term/tri-fpga-txhold/", "term/tri-fpga-rxcap/", "term/tri-fpga-rxer/", "term/tri-fpga-steps/", "term/tri-fpga-ioclients/", "term/tri-fpga-claim/", "term/tri-fpga-release/", "term/tri-fpga-next/", "term/t27c-memory/", "term/t27c-spi-tb/", "term/t27c-check-uart/", "term/t27c-check-spi/", "term/t27c-check-apb/", "term/t27c-check-axi4/", "term/t27c-check-bridge/", "term/t27c-check-memory/", "term/t27c-verilog-uart/", "term/t27c-verilog-spi/", "term/t27c-verilog-apb/", "term/t27c-verilog-axi4/", "term/t27c-verilog-bridge/", "term/t27c-verilog-top/", "term/t27c-verilog-uart-tb/", "term/t27c-verilog-apb-tb/", "term/t27c-verilog-axi-tb/", "term/t27c-verilog-mem-tb/", "term/t27c-verilog-integration-tb/", "term/t27c-verilog-top-tb/", "term/t27c-verilog-spi-tb/", "term/t27c-hir-uart/", "term/t27c-hir-apb/", "term/t27c-hir-axi4/", "term/t27c-eth-crc/", "term/t27c-rgmii/", "widgets/jtag-verdict/", "widgets/line-bounce/", "widgets/eye-isi/", "widgets/ecc-secded/", "widgets/carry-chain/", "widgets/prefix-adder/", "widgets/csa-tree/", "widgets/shift-add/", "widgets/booth-radix4/", "widgets/dsp-fit/", "widgets/q-format/", "widgets/round-sat/", "widgets/sqnr/", "widgets/cordic/", "widgets/cordic-vector/", "widgets/sine-table/", "widgets/alias-fold/", "widgets/nco/", "widgets/dft-bin/", "widgets/fir-response/", "widgets/fir-design/", "widgets/fir-cost/", "widgets/decimator/", "widgets/cic-growth/", "widgets/polyphase/", "widgets/fft-butterfly/", "widgets/bit-reverse/", "widgets/fft-scaling/", "widgets/barker-corr/", "widgets/fir-fit/", "widgets/fir-on-board/", "play/hello-world/", "play/hello-world/rust/", "play/hello-world/js/", "term/t27c-hello-world/", "term/t27-tri-misread/", "widgets/test-touch/", "widgets/t27c-stages/", "term/t27c-e8m0/", "play/e8m0/", "term/t27c-ocp-mx/", "play/gfternary/", "term/t27c-gfternary/", "play/golden-sieve/", "term/t27c-golden-sieve/", "play/tnf17/", "term/t27c-tnf17/", "term/t27c-activation-quantizer/", "term/t27c-bitnet-majority/", "term/t27c-bitnet-neuron-nchunk/", "term/t27c-comb-ternary-dot/", "term/t27c-stream-ternary-mac/", "term/t27c-bitnet-mlp/", "term/t27c-gft-smul/", "term/t27c-gft-sadd/", "term/t27c-gft-signed-mac/", "term/t27c-gft-relu/", "term/t27c-gft-exp2/", "term/t27c-gft-argmax4/", "term/t27c-gft-nll/", "term/t27c-gft-sgd-step/", "term/t27c-gft-xornet/", "widgets/basics-what-a-spec-is/", "widgets/basics-the-module-line/", "widgets/basics-comments-and-prose/", "widgets/basics-constants/", "widgets/basics-whole-numbers/", "widgets/basics-true-false-and-text/", "widgets/basics-arrays/", "widgets/basics-trits/", "widgets/basics-expressions/", "widgets/basics-a-test-block/", "widgets/basics-many-tests/", "widgets/basics-functions/", "widgets/basics-local-names/", "widgets/basics-if-and-else/", "widgets/basics-loops/", "widgets/basics-pub-or-private/", "widgets/basics-structs/", "widgets/basics-enums/", "widgets/basics-use-other-modules/", "widgets/basics-invariants/", "widgets/basics-gen-ts/", "widgets/basics-seven-backends/", "widgets/basics-stages-of-t27c/", "widgets/basics-tri-commands/", "widgets/basics-reading-errors/", "widgets/basics-a-small-program/", "widgets/basics-where-next/", "widgets/golden-split/", "widgets/golden-phi-ratio/", "widgets/golden-lucas/", "widgets/golden-split-optimality/", "widgets/golden-radix-economy/", "widgets/golden-phi-in-gf16/", "widgets/golden-gf4/", "widgets/golden-gf6/", "widgets/golden-gf8/", "widgets/golden-gf10/", "widgets/golden-gf12/", "widgets/golden-gf14/", "widgets/golden-gf16/", "widgets/golden-dot-two/", "widgets/golden-gf24/", "widgets/golden-gf32/", "widgets/golden-gf48/", "widgets/golden-gf64/", "widgets/golden-gf96/", "widgets/golden-gf128/", "widgets/golden-gf256/", "widgets/golden-gf512/", "widgets/golden-gf1024/", "widgets/golden-gft8/", "widgets/golden-gft16/", "widgets/golden-gft32/", "widgets/golden-decode/", "term/t27-tri-vectors/", "term/t27-tri-census/", "term/t27-tri-seals/", "term/t27-tri-prose/", "term/t27-tri-lean/", "term/tri-selftest/", "term/play-terminal-gates/", "widgets/tc-why-three/", "widgets/tc-balanced-count/", "widgets/tc-flip-and-cut/", "widgets/tc-kleene-gates/", "widgets/tc-consensus-any/", "widgets/tc-sum-and-carry/", "widgets/tc-half-adder/", "widgets/tc-full-adder/", "widgets/tc-ripple-carry/", "widgets/tc-times-a-trit/", "widgets/tc-long-multiply/", "widgets/tc-ternary-mac/", "widgets/tc-two-bits/", "widgets/tc-five-in-a-byte/", "widgets/tc-trit-word/", "widgets/tc-instruction-word/", "widgets/tc-opcode-map/", "widgets/tc-immediate/", "widgets/tc-ternary-alu/", "widgets/tc-three-way-jump/", "widgets/tc-run-a-program/", "widgets/tc-compare/", "widgets/tc-thirds/", "widgets/tc-sort-three/", "widgets/tc-one-neuron/", "widgets/tc-detector/", "widgets/tc-tiny-layer/", "wars", "term/tri-game-tick/", "term/tri-game-vault/", "term/perf2-knockouts/", "widgets/race-the-bee/", "widgets/gatle/", "term/tri-blog-list/", "term/tri-game-selftest/"];
-pub const WIDGET_TITLES : [252]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "WARS: agent configurations compared on real GitHub issues", "", "", "", "", "", "", ""];
-pub const WIDGET_LINES : [252]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "One issue, one pinned commit, the same tools and gates; only the decision layer changes, and a missing run stays unknown.", "", "", "", "", "", "", ""];
+pub const WIDGET_COUNT : u16 = 279;
+pub const WIDGET_IDS : [279]str = ["x7-board", "tri-x7-fasm", "tri-x7-mutate", "perf2-loop", "tri-fpga-rgmii", "tri-fpga-jtag", "tri-fpga-pins", "devkit-flow", "play-verilog", "bit-diff", "build-receipt", "cost-curves", "gf16-calc", "idcode", "lut-explain", "pin-map", "placement-map", "resource-bars", "slack-waterfall", "test-waves", "synth-smells", "bit-autopsy", "fasm-skyline", "vcd-wrapped", "fits-on", "stat-card", "lut-depth", "fsm-sketch", "lut-treemap", "t27-cell", "t27-vs-nvfp4", "uart-bucket", "tri-fpga-specs", "tri-fpga-tools", "tri-fpga-wire", "tri-fpga-batch", "tri-fpga-status", "tri-fpga-tmpcheck", "tri-fpga-keycheck", "tri-fpga-selftest", "tri-x7-refs", "tri-x7-audit", "t27c-clock-domain", "t27c-clocks", "t27c-wukong-v1", "t27c-cts", "t27c-mmcm", "t27c-mmcm-mutate", "t27c-timing", "t27c-reset-sync", "t27c-reset-sync-gen", "t27c-mtbf", "t27c-clock-domain-gen", "t27c-fifo", "t27c-fifo-mutate", "t27c-fifo-flags", "t27c-emitter-xdc", "t27c-fpga-rxcap", "t27-tri-competitors", "t27-tri-discard", "tri-fpga-txhold", "tri-fpga-rxcap", "tri-fpga-rxer", "tri-fpga-steps", "tri-fpga-ioclients", "tri-fpga-claim", "tri-fpga-release", "tri-fpga-next", "t27c-memory", "t27c-spi-tb", "t27c-check-uart", "t27c-check-spi", "t27c-check-apb", "t27c-check-axi4", "t27c-check-bridge", "t27c-check-memory", "t27c-verilog-uart", "t27c-verilog-spi", "t27c-verilog-apb", "t27c-verilog-axi4", "t27c-verilog-bridge", "t27c-verilog-top", "t27c-verilog-uart-tb", "t27c-verilog-apb-tb", "t27c-verilog-axi-tb", "t27c-verilog-mem-tb", "t27c-verilog-integration-tb", "t27c-verilog-top-tb", "t27c-verilog-spi-tb", "t27c-hir-uart", "t27c-hir-apb", "t27c-hir-axi4", "t27c-eth-crc", "t27c-rgmii", "jtag-verdict", "line-bounce", "eye-isi", "ecc-secded", "carry-chain", "prefix-adder", "csa-tree", "shift-add", "booth-radix4", "dsp-fit", "q-format", "round-sat", "sqnr", "cordic", "cordic-vector", "sine-table", "alias-fold", "nco", "dft-bin", "fir-response", "fir-design", "fir-cost", "decimator", "cic-growth", "polyphase", "fft-butterfly", "bit-reverse", "fft-scaling", "barker-corr", "fir-fit", "fir-on-board", "play-hello-world", "play-rust", "play-js", "t27c-hello-world", "t27-tri-misread", "test-touch", "t27c-stages", "t27c-e8m0", "play-e8m0", "t27c-ocp-mx", "play-gfternary", "t27c-gfternary", "play-golden-sieve", "t27c-golden-sieve", "play-tnf17", "t27c-tnf17", "t27c-activation-quantizer", "t27c-bitnet-majority", "t27c-bitnet-neuron-nchunk", "t27c-comb-ternary-dot", "t27c-stream-ternary-mac", "t27c-bitnet-mlp", "t27c-gft-smul", "t27c-gft-sadd", "t27c-gft-signed-mac", "t27c-gft-relu", "t27c-gft-exp2", "t27c-gft-argmax4", "t27c-gft-nll", "t27c-gft-sgd-step", "t27c-gft-xornet", "basics-what-a-spec-is", "basics-the-module-line", "basics-comments-and-prose", "basics-constants", "basics-whole-numbers", "basics-true-false-and-text", "basics-arrays", "basics-trits", "basics-expressions", "basics-a-test-block", "basics-many-tests", "basics-functions", "basics-local-names", "basics-if-and-else", "basics-loops", "basics-pub-or-private", "basics-structs", "basics-enums", "basics-use-other-modules", "basics-invariants", "basics-gen-ts", "basics-seven-backends", "basics-stages-of-t27c", "basics-tri-commands", "basics-reading-errors", "basics-a-small-program", "basics-where-next", "golden-split", "golden-phi-ratio", "golden-lucas", "golden-split-optimality", "golden-radix-economy", "golden-phi-in-gf16", "golden-gf4", "golden-gf6", "golden-gf8", "golden-gf10", "golden-gf12", "golden-gf14", "golden-gf16", "golden-dot-two", "golden-gf24", "golden-gf32", "golden-gf48", "golden-gf64", "golden-gf96", "golden-gf128", "golden-gf256", "golden-gf512", "golden-gf1024", "golden-gft8", "golden-gft16", "golden-gft32", "golden-decode", "t27-tri-vectors", "t27-tri-census", "t27-tri-seals", "t27-tri-prose", "t27-tri-lean", "tri-selftest", "play-terminal-gates", "tc-why-three", "tc-balanced-count", "tc-flip-and-cut", "tc-kleene-gates", "tc-consensus-any", "tc-sum-and-carry", "tc-half-adder", "tc-full-adder", "tc-ripple-carry", "tc-times-a-trit", "tc-long-multiply", "tc-ternary-mac", "tc-two-bits", "tc-five-in-a-byte", "tc-trit-word", "tc-instruction-word", "tc-opcode-map", "tc-immediate", "tc-ternary-alu", "tc-three-way-jump", "tc-run-a-program", "tc-compare", "tc-thirds", "tc-sort-three", "tc-one-neuron", "tc-detector", "tc-tiny-layer", "nn-relu", "nn-sigmoid", "nn-tanh", "nn-gelu", "nn-silu", "nn-elu", "nn-softmax", "nn-embedding", "nn-dropout", "nn-conv", "nn-maxpool", "nn-batchnorm", "nn-mse", "nn-bce", "nn-huber", "nn-sgd", "nn-rmsprop", "nn-adam", "nn-backprop", "nn-xor", "nn-lamb", "nn-rnn", "nn-lstm", "nn-attention", "nn-positions", "nn-encoder", "nn-ppo", "wars", "tri-game-tick", "tri-game-vault", "perf2-knockouts", "race-the-bee", "gatle", "tri-blog-list", "tri-game-selftest"];
+pub const WIDGET_KINDS : [279]str = ["cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "player", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "player", "player", "player", "cast", "cast", "tool", "tool", "cast", "player", "cast", "player", "cast", "player", "cast", "player", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tab", "cast", "cast", "cast", "tool", "tool", "cast", "cast"];
+pub const WIDGET_CATEGORIES : [279]str = ["fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "game", "game", "game", "game", "game", "game", "game", "game"];
+pub const WIDGET_PAGES : [279]str = ["term/x7-board/", "term/tri-x7-fasm/", "term/tri-x7-mutate/", "term/perf2-loop/", "term/tri-fpga-rgmii/", "term/tri-fpga-jtag/", "term/tri-fpga-pins/", "term/devkit-flow/", "play/hello-world/verilog/", "widgets/bit-diff/", "widgets/build-receipt/", "widgets/cost-curves/", "widgets/gf16-calc/", "widgets/idcode/", "widgets/lut-explain/", "widgets/pin-map/", "widgets/placement-map/", "widgets/resource-bars/", "widgets/slack-waterfall/", "widgets/test-waves/", "widgets/synth-smells/", "widgets/bit-autopsy/", "widgets/fasm-skyline/", "widgets/vcd-wrapped/", "widgets/fits-on/", "widgets/stat-card/", "widgets/lut-depth/", "widgets/fsm-sketch/", "widgets/lut-treemap/", "widgets/t27-cell/", "widgets/t27-vs-nvfp4/", "widgets/uart-bucket/", "term/tri-fpga-specs/", "term/tri-fpga-tools/", "term/tri-fpga-wire/", "term/tri-fpga-batch/", "term/tri-fpga-status/", "term/tri-fpga-tmpcheck/", "term/tri-fpga-keycheck/", "term/tri-fpga-selftest/", "term/tri-x7-refs/", "term/tri-x7-audit/", "term/t27c-clock-domain/", "term/t27c-clocks/", "term/t27c-wukong-v1/", "term/t27c-cts/", "term/t27c-mmcm/", "term/t27c-mmcm-mutate/", "term/t27c-timing/", "term/t27c-reset-sync/", "term/t27c-reset-sync-gen/", "term/t27c-mtbf/", "term/t27c-clock-domain-gen/", "term/t27c-fifo/", "term/t27c-fifo-mutate/", "term/t27c-fifo-flags/", "term/t27c-emitter-xdc/", "term/t27c-fpga-rxcap/", "term/t27-tri-competitors/", "term/t27-tri-discard/", "term/tri-fpga-txhold/", "term/tri-fpga-rxcap/", "term/tri-fpga-rxer/", "term/tri-fpga-steps/", "term/tri-fpga-ioclients/", "term/tri-fpga-claim/", "term/tri-fpga-release/", "term/tri-fpga-next/", "term/t27c-memory/", "term/t27c-spi-tb/", "term/t27c-check-uart/", "term/t27c-check-spi/", "term/t27c-check-apb/", "term/t27c-check-axi4/", "term/t27c-check-bridge/", "term/t27c-check-memory/", "term/t27c-verilog-uart/", "term/t27c-verilog-spi/", "term/t27c-verilog-apb/", "term/t27c-verilog-axi4/", "term/t27c-verilog-bridge/", "term/t27c-verilog-top/", "term/t27c-verilog-uart-tb/", "term/t27c-verilog-apb-tb/", "term/t27c-verilog-axi-tb/", "term/t27c-verilog-mem-tb/", "term/t27c-verilog-integration-tb/", "term/t27c-verilog-top-tb/", "term/t27c-verilog-spi-tb/", "term/t27c-hir-uart/", "term/t27c-hir-apb/", "term/t27c-hir-axi4/", "term/t27c-eth-crc/", "term/t27c-rgmii/", "widgets/jtag-verdict/", "widgets/line-bounce/", "widgets/eye-isi/", "widgets/ecc-secded/", "widgets/carry-chain/", "widgets/prefix-adder/", "widgets/csa-tree/", "widgets/shift-add/", "widgets/booth-radix4/", "widgets/dsp-fit/", "widgets/q-format/", "widgets/round-sat/", "widgets/sqnr/", "widgets/cordic/", "widgets/cordic-vector/", "widgets/sine-table/", "widgets/alias-fold/", "widgets/nco/", "widgets/dft-bin/", "widgets/fir-response/", "widgets/fir-design/", "widgets/fir-cost/", "widgets/decimator/", "widgets/cic-growth/", "widgets/polyphase/", "widgets/fft-butterfly/", "widgets/bit-reverse/", "widgets/fft-scaling/", "widgets/barker-corr/", "widgets/fir-fit/", "widgets/fir-on-board/", "play/hello-world/", "play/hello-world/rust/", "play/hello-world/js/", "term/t27c-hello-world/", "term/t27-tri-misread/", "widgets/test-touch/", "widgets/t27c-stages/", "term/t27c-e8m0/", "play/e8m0/", "term/t27c-ocp-mx/", "play/gfternary/", "term/t27c-gfternary/", "play/golden-sieve/", "term/t27c-golden-sieve/", "play/tnf17/", "term/t27c-tnf17/", "term/t27c-activation-quantizer/", "term/t27c-bitnet-majority/", "term/t27c-bitnet-neuron-nchunk/", "term/t27c-comb-ternary-dot/", "term/t27c-stream-ternary-mac/", "term/t27c-bitnet-mlp/", "term/t27c-gft-smul/", "term/t27c-gft-sadd/", "term/t27c-gft-signed-mac/", "term/t27c-gft-relu/", "term/t27c-gft-exp2/", "term/t27c-gft-argmax4/", "term/t27c-gft-nll/", "term/t27c-gft-sgd-step/", "term/t27c-gft-xornet/", "widgets/basics-what-a-spec-is/", "widgets/basics-the-module-line/", "widgets/basics-comments-and-prose/", "widgets/basics-constants/", "widgets/basics-whole-numbers/", "widgets/basics-true-false-and-text/", "widgets/basics-arrays/", "widgets/basics-trits/", "widgets/basics-expressions/", "widgets/basics-a-test-block/", "widgets/basics-many-tests/", "widgets/basics-functions/", "widgets/basics-local-names/", "widgets/basics-if-and-else/", "widgets/basics-loops/", "widgets/basics-pub-or-private/", "widgets/basics-structs/", "widgets/basics-enums/", "widgets/basics-use-other-modules/", "widgets/basics-invariants/", "widgets/basics-gen-ts/", "widgets/basics-seven-backends/", "widgets/basics-stages-of-t27c/", "widgets/basics-tri-commands/", "widgets/basics-reading-errors/", "widgets/basics-a-small-program/", "widgets/basics-where-next/", "widgets/golden-split/", "widgets/golden-phi-ratio/", "widgets/golden-lucas/", "widgets/golden-split-optimality/", "widgets/golden-radix-economy/", "widgets/golden-phi-in-gf16/", "widgets/golden-gf4/", "widgets/golden-gf6/", "widgets/golden-gf8/", "widgets/golden-gf10/", "widgets/golden-gf12/", "widgets/golden-gf14/", "widgets/golden-gf16/", "widgets/golden-dot-two/", "widgets/golden-gf24/", "widgets/golden-gf32/", "widgets/golden-gf48/", "widgets/golden-gf64/", "widgets/golden-gf96/", "widgets/golden-gf128/", "widgets/golden-gf256/", "widgets/golden-gf512/", "widgets/golden-gf1024/", "widgets/golden-gft8/", "widgets/golden-gft16/", "widgets/golden-gft32/", "widgets/golden-decode/", "term/t27-tri-vectors/", "term/t27-tri-census/", "term/t27-tri-seals/", "term/t27-tri-prose/", "term/t27-tri-lean/", "term/tri-selftest/", "term/play-terminal-gates/", "widgets/tc-why-three/", "widgets/tc-balanced-count/", "widgets/tc-flip-and-cut/", "widgets/tc-kleene-gates/", "widgets/tc-consensus-any/", "widgets/tc-sum-and-carry/", "widgets/tc-half-adder/", "widgets/tc-full-adder/", "widgets/tc-ripple-carry/", "widgets/tc-times-a-trit/", "widgets/tc-long-multiply/", "widgets/tc-ternary-mac/", "widgets/tc-two-bits/", "widgets/tc-five-in-a-byte/", "widgets/tc-trit-word/", "widgets/tc-instruction-word/", "widgets/tc-opcode-map/", "widgets/tc-immediate/", "widgets/tc-ternary-alu/", "widgets/tc-three-way-jump/", "widgets/tc-run-a-program/", "widgets/tc-compare/", "widgets/tc-thirds/", "widgets/tc-sort-three/", "widgets/tc-one-neuron/", "widgets/tc-detector/", "widgets/tc-tiny-layer/", "widgets/nn-relu/", "widgets/nn-sigmoid/", "widgets/nn-tanh/", "widgets/nn-gelu/", "widgets/nn-silu/", "widgets/nn-elu/", "widgets/nn-softmax/", "widgets/nn-embedding/", "widgets/nn-dropout/", "widgets/nn-conv/", "widgets/nn-maxpool/", "widgets/nn-batchnorm/", "widgets/nn-mse/", "widgets/nn-bce/", "widgets/nn-huber/", "widgets/nn-sgd/", "widgets/nn-rmsprop/", "widgets/nn-adam/", "widgets/nn-backprop/", "widgets/nn-xor/", "widgets/nn-lamb/", "widgets/nn-rnn/", "widgets/nn-lstm/", "widgets/nn-attention/", "widgets/nn-positions/", "widgets/nn-encoder/", "widgets/nn-ppo/", "wars", "term/tri-game-tick/", "term/tri-game-vault/", "term/perf2-knockouts/", "widgets/race-the-bee/", "widgets/gatle/", "term/tri-blog-list/", "term/tri-game-selftest/"];
+pub const WIDGET_TITLES : [279]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "WARS: agent configurations compared on real GitHub issues", "", "", "", "", "", "", ""];
+pub const WIDGET_LINES : [279]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "One issue, one pinned commit, the same tools and gates; only the decision layer changes, and a missing run stays unknown.", "", "", "", "", "", "", ""];
; Why a person would pass this one on: the reason the widget is in the gallery at all.
-pub const WIDGET_HOOKS : [252]str = ["The bitstream is byte-identical to openXC7's and runs on a real board: the claim most FPGA people doubt, shown.", "A Xilinx 7-series bitstream assembled with no Vivado anywhere in the path.", "Plant a bug in the bitstream path and watch the tests catch it: mutation testing for hardware.", "Place and route to a loaded bitstream, with the clock on screen.", "RGMII hold margins read from nextpnr's own SDF, not from a datasheet.", "The cable asks which chip is there and the chip answers with its IDCODE.", "Before an edit to a Verilog file: every spec and generator it would touch.", "The whole FPGA flow, one layer per line, from spec to board.", "A spec turned into Verilog inside a post, with the RTL read the way a hardware engineer reads it.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "17 of 17 conformance specs compile, their test blocks pass, and no generated file has drifted.", "14 of 14 bench tools pass their own self-tests in one sweep: decode 18/18, repin 17/17, firelog 10/10, tmpcheck 9/9, stamps 9/9, keep 9/9.", "Requested 115,200 puts 115,385 on the wire (+0.16 %), 921,600 puts 923,077, and 3,000,000 is exact.", "26 checks, no board, no RTL: every receipt returns, 11 loss and fault cases recover, and a lie stops the run.", "One command reads the bench: cables, locks, and the last board runs: three x7-board runs PASS 51840/51840.", "112 files under /tmp pinned by sha256 in specs: every one has a kept copy, 0 lost, 0 unlocated.", "84 board and model logs scanned for the TRI-NET key: 0 hits, 0 logs of 1 MB or more.", "16 spec-named scripts run their own --self-test, 0 bad; 8 without the flag are listed, not skipped silently.", "The part table in far.t27 matches prjxray-db part.json, all 6 reference bitstreams present, bitwalk reads all 6 in 84.4 ms.", "One command re-runs every sweep: CRC 12/12, ECC 32520/32520 frames, 82/82 pins, 4224/4224 segbits, gold 5/5. audit: CLEAN.", "The CDC spec through the real compiler: 12 tests pass natively and the header says what the model does and does not claim.", "The tri clocks card as a spec: constants a tool page reads, and a native test-report that says so plainly -- 0 tests, because it is not a test suite.", "The board itself as a spec: 15 tests pass natively and the part line names the xc7a200tfgg676 the Wukong carries.", "Clock-tree synthesis in numbers: 15 tests pass natively, and the header pins the Artix-7 teaching delays the tree arithmetic uses.", "The MMCM model through the real compiler: VCO window, M/D/O ranges, fractional multiply in eighths, the 1/56 phase step, 16 tests natively.", "Widen the VCO window with one sed and exactly one test objects -- the rejection test that names the 500 MHz config it calls out.", "Static timing as arithmetic: 25 tests pass natively on arcs, slack and Fmax, with the Artix-7 teaching delays in the header.", "The three reset kinds through the real compiler: async assert with sync release through 2 to 4 flops, and the racy async release, 16 tests natively.", "What the compiler emits for the reset pipe: t27c gen prints the Zig, one struct copy per clock, and wc -l counts it.", "Metastability MTBF without floats: the WP323 form in Q10 log2, one flop negative slack, two flops astronomical, 15 tests natively.", "The CDC model as Zig: t27c gen prints what it compiled and wc -l says how many lines of code the model is.", "The FIFO stdlib through the real compiler: 18 tests pass natively and 6 invariants are proved comptime.", "Move a pointer by 3 instead of 1 and every test still passes: the suite asserts fill and flags, never pointer values.", "Flip one flag assignment and a single test objects: the guard that keeps the FIFO honest.", "The pins-to-XDC emitter spec parsed by the real compiler, and the create_clock lines it exists to write.", "The fpga-rxcap tool card as a spec: constants for a tool page, and a native run that says 0 tests and why.", "168 records: 14 papers entered twice, 141 cite no score, 144 state zero at pass@1.", "67 specs lose 23,578 tokens to parser recovery, each held at its pinned bound.", "Smallest hold 3445 ps on TXD1 against a 1000 ps bench; each TX pin setup and hold from the routed SDF.", "Per-bit setup and hold of an async RX nibble against its RXC edge, from the routed SDF: one nibble changes 2387 to 2486 ps apart, so a binary counter tears where a gray one cannot.", "RX_ER pulses injected at exact nibble positions of a ping -- preamble, mid-frame, IP header -- and the frame state each pulse leaves behind, RTL vs model.", "The Ethernet bring-up plan on the bench: pre-registered steps, logged exits, what ran and what did not.", "The registered IO clients on the bench, read live: names, claims and what each declares it owns.", "The bench claim in action: one client takes the hardware lock and the tool reports who holds it.", "The bench lock given back: the client gives up its claim and the bench reports itself free.", "The next queued step on the bench, read live from the run state.", "The memory map spec through the real compiler: 15 tests pass natively and 6 invariants are proved comptime.", "The SPI testbench through the real compiler: 7 tests pass natively, driving the modes and checking the captures.", "t27c check on the UART spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the SPI spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the APB bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the AXI4 spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the bus bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the memory spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "The UART spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The SPI spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The APB bridge spec lowered to synthesizable Verilog: port list, PSEL and PENABLE wiring, the address decode.", "The AXI4 spec lowered to synthesizable Verilog: the five channels' wiring and handshakes as generated wires.", "The bus bridge spec lowered to synthesizable Verilog: both sides of the crossing as generated wires.", "The whole design in one spec lowered to synthesizable Verilog: clocks, resets and bus wiring as the compiler emits them.", "The UART testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The APB bridge testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The AXI4 testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The memory testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The integration testbench spec lowered to Verilog: every bus of the design in one bench.", "The top-level testbench spec lowered to Verilog: the whole design under one bench.", "The SPI testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The UART spec lowered to hardware IR: the compiler's own view of the wires and state before Verilog exists.", "The APB bridge spec lowered to hardware IR: PSEL, PENABLE and the decode as the compiler structures them.", "The AXI4 spec lowered to hardware IR: the five channels and their handshakes as the compiler structures them.", "The Ethernet FCS spec through the real compiler: 8 tests pass natively -- the table derived from one bit step, the 0xCBF43926 check value, the 0xDEBB20E3 receiver residue.", "The RGMII spec through the real compiler: 7 tests pass natively and 1 invariant is proved comptime -- the 8 ns gigabit period, a nibble on both edges, the 78 ps tap ladder.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "The t27 compiler running as WebAssembly in a post: compile, test, seven backends.", "The same spec as Rust, printed by a terminal in the page.", "The only backend that runs in the browser: generate the module, then run it.", "t27c on hello_world.t27, on a real machine, every byte as printed.", "Code that compiles and is wrong: the tool that finds it.", "", "", "The OCP MX scale byte as a spec: 2^(e-127), code 255 is NaN, 18 tests and 9 comptime invariants, compiled on a real machine.", "The same scale byte compiled in your browser: seven backends, the spec's tests run in the page, and every skip says why.", "One outlier sets the scale for 32 weights: MXFP4 flushes 23 of them to zero, MXFP8 E4M3 none. 49 tests pass; a planted bug fails one.", "Ternary weights scaled by phi, compiled in your browser: codes 00, 01, 10, and the fourth code folded to zero.", "13 of 13 tests pass natively, and the report says 8 of them executed no assert; a planted bug fails exactly one.", "Five measured rules for a ternary weight alphabet, and the one formula that passes all five, run in your browser.", "One more trit and the spec does not compile at all: BLOCKED, not failed, because no test ever ran.", "A 17-bit float whose exponent is four balanced trits, 3^4 = 81 values, compiled in your browser.", "34 tests pass natively, none vacuous; swap one XOR for OR and only the test that negates twice fails.", "A neuron's sum squeezed back to one trit: 7 tests pass natively, and turning one > into >= fails only the boundary test.", "One ternary neuron, two functions picked by its weights: 12 tests pass natively, and one weight moved from P to Z fails exactly one.", "A neuron wider than 27 inputs, summed chunk by chunk: 11 tests pass natively, and a loop that stops after one chunk fails exactly one.", "A 27-lane ternary dot product as pure wires: 4 tests pass natively, and one wrong sign in tmul fails exactly one.", "The same dot product added into a register on every clock: 4 tests pass natively, and a lane mask short one bit fails exactly one.", "A whole small ternary network, 3 neurons then 2: 4 tests pass natively, and a trit packed one bit low fails exactly one.", "A signed multiply: 3 tests pass natively, none vacuous, and a sign stuck at 0 fails exactly one, m2.", "A signed add: 3 tests pass natively, and adding where opposite signs must subtract fails exactly one, a2.", "Two signed products summed: 4 tests pass natively, and OR in place of the sign XOR fails exactly one, pp.", "A ReLU: 4 tests pass natively, and letting negative inputs through fails only negz, the one test with a negative input.", "2^x for a softmax: 4 tests pass natively, and dropping the minus sign of k fails exactly one, em1.", "argmax by comparing bits alone: 4 tests pass natively, and >= in place of > fails exactly one, tie_low.", "A loss in bits: 3 tests pass natively, and a zero with the sign bit set fails exactly one, perfect.", "One SGD step: 3 tests pass natively, and g times g in place of eta times g fails exactly one, ascend.", "A 2-layer ReLU net for XOR: 4 tests pass natively, and one relu removed fails exactly one, x00.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "34 vector files, 512 cases: 2 files executed, 8 debt, 24 prose-only; 19% carry data.", "Four census populations recounted another way all agree; three are named as uncheckable.", "31 of 1,428 seals record no output (16 specs); 3 of those are missing from the debt ledger.", "11 specs refused at parse, 0 blocked only by prose, 23 broken on purpose as fixtures.", "12 Lean files, 15,553 lines and 4 sorry, are reached by no build root: nothing compiles them.", "tri, railway, python3 and the loop directory: each present or not, in one call.", "The terminal for seven t27 backends: every command run on hello_world. 21 pass, 0 fail.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "Pick a lane and see whether the TRI decision layer beats the baseline on the same issue.", "One tick of the Queen's loop: an anomaly scan that reports only what changed.", "The loop's skills checked against snapshots before an agent may use them.", "Three performance bits, and one knockout bitstream per bit to prove each one matters.", "", "", "73 published posts with date, language and revision, newest first.", "6 of 6 self-tests hold: anomalies, blog_drift, lessons, site_gate, tick, vault."];
+pub const WIDGET_HOOKS : [279]str = ["The bitstream is byte-identical to openXC7's and runs on a real board: the claim most FPGA people doubt, shown.", "A Xilinx 7-series bitstream assembled with no Vivado anywhere in the path.", "Plant a bug in the bitstream path and watch the tests catch it: mutation testing for hardware.", "Place and route to a loaded bitstream, with the clock on screen.", "RGMII hold margins read from nextpnr's own SDF, not from a datasheet.", "The cable asks which chip is there and the chip answers with its IDCODE.", "Before an edit to a Verilog file: every spec and generator it would touch.", "The whole FPGA flow, one layer per line, from spec to board.", "A spec turned into Verilog inside a post, with the RTL read the way a hardware engineer reads it.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "17 of 17 conformance specs compile, their test blocks pass, and no generated file has drifted.", "14 of 14 bench tools pass their own self-tests in one sweep: decode 18/18, repin 17/17, firelog 10/10, tmpcheck 9/9, stamps 9/9, keep 9/9.", "Requested 115,200 puts 115,385 on the wire (+0.16 %), 921,600 puts 923,077, and 3,000,000 is exact.", "26 checks, no board, no RTL: every receipt returns, 11 loss and fault cases recover, and a lie stops the run.", "One command reads the bench: cables, locks, and the last board runs: three x7-board runs PASS 51840/51840.", "112 files under /tmp pinned by sha256 in specs: every one has a kept copy, 0 lost, 0 unlocated.", "84 board and model logs scanned for the TRI-NET key: 0 hits, 0 logs of 1 MB or more.", "16 spec-named scripts run their own --self-test, 0 bad; 8 without the flag are listed, not skipped silently.", "The part table in far.t27 matches prjxray-db part.json, all 6 reference bitstreams present, bitwalk reads all 6 in 84.4 ms.", "One command re-runs every sweep: CRC 12/12, ECC 32520/32520 frames, 82/82 pins, 4224/4224 segbits, gold 5/5. audit: CLEAN.", "The CDC spec through the real compiler: 12 tests pass natively and the header says what the model does and does not claim.", "The tri clocks card as a spec: constants a tool page reads, and a native test-report that says so plainly -- 0 tests, because it is not a test suite.", "The board itself as a spec: 15 tests pass natively and the part line names the xc7a200tfgg676 the Wukong carries.", "Clock-tree synthesis in numbers: 15 tests pass natively, and the header pins the Artix-7 teaching delays the tree arithmetic uses.", "The MMCM model through the real compiler: VCO window, M/D/O ranges, fractional multiply in eighths, the 1/56 phase step, 16 tests natively.", "Widen the VCO window with one sed and exactly one test objects -- the rejection test that names the 500 MHz config it calls out.", "Static timing as arithmetic: 25 tests pass natively on arcs, slack and Fmax, with the Artix-7 teaching delays in the header.", "The three reset kinds through the real compiler: async assert with sync release through 2 to 4 flops, and the racy async release, 16 tests natively.", "What the compiler emits for the reset pipe: t27c gen prints the Zig, one struct copy per clock, and wc -l counts it.", "Metastability MTBF without floats: the WP323 form in Q10 log2, one flop negative slack, two flops astronomical, 15 tests natively.", "The CDC model as Zig: t27c gen prints what it compiled and wc -l says how many lines of code the model is.", "The FIFO stdlib through the real compiler: 18 tests pass natively and 6 invariants are proved comptime.", "Move a pointer by 3 instead of 1 and every test still passes: the suite asserts fill and flags, never pointer values.", "Flip one flag assignment and a single test objects: the guard that keeps the FIFO honest.", "The pins-to-XDC emitter spec parsed by the real compiler, and the create_clock lines it exists to write.", "The fpga-rxcap tool card as a spec: constants for a tool page, and a native run that says 0 tests and why.", "168 records: 14 papers entered twice, 141 cite no score, 144 state zero at pass@1.", "67 specs lose 23,578 tokens to parser recovery, each held at its pinned bound.", "Smallest hold 3445 ps on TXD1 against a 1000 ps bench; each TX pin setup and hold from the routed SDF.", "Per-bit setup and hold of an async RX nibble against its RXC edge, from the routed SDF: one nibble changes 2387 to 2486 ps apart, so a binary counter tears where a gray one cannot.", "RX_ER pulses injected at exact nibble positions of a ping -- preamble, mid-frame, IP header -- and the frame state each pulse leaves behind, RTL vs model.", "The Ethernet bring-up plan on the bench: pre-registered steps, logged exits, what ran and what did not.", "The registered IO clients on the bench, read live: names, claims and what each declares it owns.", "The bench claim in action: one client takes the hardware lock and the tool reports who holds it.", "The bench lock given back: the client gives up its claim and the bench reports itself free.", "The next queued step on the bench, read live from the run state.", "The memory map spec through the real compiler: 15 tests pass natively and 6 invariants are proved comptime.", "The SPI testbench through the real compiler: 7 tests pass natively, driving the modes and checking the captures.", "t27c check on the UART spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the SPI spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the APB bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the AXI4 spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the bus bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the memory spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "The UART spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The SPI spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The APB bridge spec lowered to synthesizable Verilog: port list, PSEL and PENABLE wiring, the address decode.", "The AXI4 spec lowered to synthesizable Verilog: the five channels' wiring and handshakes as generated wires.", "The bus bridge spec lowered to synthesizable Verilog: both sides of the crossing as generated wires.", "The whole design in one spec lowered to synthesizable Verilog: clocks, resets and bus wiring as the compiler emits them.", "The UART testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The APB bridge testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The AXI4 testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The memory testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The integration testbench spec lowered to Verilog: every bus of the design in one bench.", "The top-level testbench spec lowered to Verilog: the whole design under one bench.", "The SPI testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The UART spec lowered to hardware IR: the compiler's own view of the wires and state before Verilog exists.", "The APB bridge spec lowered to hardware IR: PSEL, PENABLE and the decode as the compiler structures them.", "The AXI4 spec lowered to hardware IR: the five channels and their handshakes as the compiler structures them.", "The Ethernet FCS spec through the real compiler: 8 tests pass natively -- the table derived from one bit step, the 0xCBF43926 check value, the 0xDEBB20E3 receiver residue.", "The RGMII spec through the real compiler: 7 tests pass natively and 1 invariant is proved comptime -- the 8 ns gigabit period, a nibble on both edges, the 78 ps tap ladder.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "The t27 compiler running as WebAssembly in a post: compile, test, seven backends.", "The same spec as Rust, printed by a terminal in the page.", "The only backend that runs in the browser: generate the module, then run it.", "t27c on hello_world.t27, on a real machine, every byte as printed.", "Code that compiles and is wrong: the tool that finds it.", "", "", "The OCP MX scale byte as a spec: 2^(e-127), code 255 is NaN, 18 tests and 9 comptime invariants, compiled on a real machine.", "The same scale byte compiled in your browser: seven backends, the spec's tests run in the page, and every skip says why.", "One outlier sets the scale for 32 weights: MXFP4 flushes 23 of them to zero, MXFP8 E4M3 none. 49 tests pass; a planted bug fails one.", "Ternary weights scaled by phi, compiled in your browser: codes 00, 01, 10, and the fourth code folded to zero.", "13 of 13 tests pass natively, and the report says 8 of them executed no assert; a planted bug fails exactly one.", "Five measured rules for a ternary weight alphabet, and the one formula that passes all five, run in your browser.", "One more trit and the spec does not compile at all: BLOCKED, not failed, because no test ever ran.", "A 17-bit float whose exponent is four balanced trits, 3^4 = 81 values, compiled in your browser.", "34 tests pass natively, none vacuous; swap one XOR for OR and only the test that negates twice fails.", "A neuron's sum squeezed back to one trit: 7 tests pass natively, and turning one > into >= fails only the boundary test.", "One ternary neuron, two functions picked by its weights: 12 tests pass natively, and one weight moved from P to Z fails exactly one.", "A neuron wider than 27 inputs, summed chunk by chunk: 11 tests pass natively, and a loop that stops after one chunk fails exactly one.", "A 27-lane ternary dot product as pure wires: 4 tests pass natively, and one wrong sign in tmul fails exactly one.", "The same dot product added into a register on every clock: 4 tests pass natively, and a lane mask short one bit fails exactly one.", "A whole small ternary network, 3 neurons then 2: 4 tests pass natively, and a trit packed one bit low fails exactly one.", "A signed multiply: 3 tests pass natively, none vacuous, and a sign stuck at 0 fails exactly one, m2.", "A signed add: 3 tests pass natively, and adding where opposite signs must subtract fails exactly one, a2.", "Two signed products summed: 4 tests pass natively, and OR in place of the sign XOR fails exactly one, pp.", "A ReLU: 4 tests pass natively, and letting negative inputs through fails only negz, the one test with a negative input.", "2^x for a softmax: 4 tests pass natively, and dropping the minus sign of k fails exactly one, em1.", "argmax by comparing bits alone: 4 tests pass natively, and >= in place of > fails exactly one, tie_low.", "A loss in bits: 3 tests pass natively, and a zero with the sign bit set fails exactly one, perfect.", "One SGD step: 3 tests pass natively, and g times g in place of eta times g fails exactly one, ascend.", "A 2-layer ReLU net for XOR: 4 tests pass natively, and one relu removed fails exactly one, x00.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "34 vector files, 512 cases: 2 files executed, 8 debt, 24 prose-only; 19% carry data.", "Four census populations recounted another way all agree; three are named as uncheckable.", "31 of 1,428 seals record no output (16 specs); 3 of those are missing from the debt ledger.", "11 specs refused at parse, 0 blocked only by prose, 23 broken on purpose as fixtures.", "12 Lean files, 15,553 lines and 4 sorry, are reached by no build root: nothing compiles them.", "tri, railway, python3 and the loop directory: each present or not, in one call.", "The terminal for seven t27 backends: every command run on hello_world. 21 pass, 0 fail.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "Pick a lane and see whether the TRI decision layer beats the baseline on the same issue.", "One tick of the Queen's loop: an anomaly scan that reports only what changed.", "The loop's skills checked against snapshots before an agent may use them.", "Three performance bits, and one knockout bitstream per bit to prove each one matters.", "", "", "73 published posts with date, language and revision, newest first.", "6 of 6 self-tests hold: anomalies, blog_drift, lessons, site_gate, tick, vault."];
; --- Ideas: asked for, not built ---------------------------------------------------------------
; From what engineers share today (gh-skyline, Tiny Tapeout renders, Wrapped cards, Gatle-style
@@ -119,9 +119,9 @@ test the_fpga_shelf_comes_first_and_opens_on_the_board {
}
test the_arena_is_a_tab_and_says_its_own_words {
- assert WIDGET_IDS[244] == "wars";
- assert WIDGET_KINDS[244] == "tab";
- assert WIDGET_PAGES[244] == "wars";
+ assert WIDGET_IDS[271] == "wars";
+ assert WIDGET_KINDS[271] == "tab";
+ assert WIDGET_PAGES[271] == "wars";
}
test the_gallery_posts_nothing {
@@ -157,8 +157,8 @@ test the_verification_shelf_is_rehearsals_and_audits {
test ideas_are_counted_apart_from_widgets {
assert IDEA_COUNT == 6;
- assert WIDGET_IDS[249] == "gatle";
- assert WIDGET_COUNT == 252;
+ assert WIDGET_IDS[276] == "gatle";
+ assert WIDGET_COUNT == 279;
}
test the_ternary_machine_shelf_runs_before_the_arena {
@@ -172,9 +172,9 @@ test every_tool_lives_under_widgets_and_keeps_its_words_on_its_page {
assert WIDGET_KINDS[9] == "tool";
assert WIDGET_PAGES[9] == "widgets/bit-diff/";
assert WIDGET_HOOKS[9] == "";
- assert WIDGET_KINDS[248] == "tool";
- assert WIDGET_CATEGORIES[248] == "game";
- assert WIDGET_PAGES[248] == "widgets/race-the-bee/";
+ assert WIDGET_KINDS[275] == "tool";
+ assert WIDGET_CATEGORIES[275] == "game";
+ assert WIDGET_PAGES[275] == "widgets/race-the-bee/";
}
test the_arithmetic_shelf_closes_the_fpga_bench {
@@ -185,3 +185,12 @@ test the_arithmetic_shelf_closes_the_fpga_bench {
assert WIDGET_CATEGORIES[124] == "fpga";
assert WIDGET_HOOKS[124] == "";
}
+
+test the_neural_network_shelf_runs_before_the_arena {
+ assert WIDGET_IDS[244] == "nn-relu";
+ assert WIDGET_KINDS[244] == "tool";
+ assert WIDGET_PAGES[244] == "widgets/nn-relu/";
+ assert WIDGET_IDS[270] == "nn-ppo";
+ assert WIDGET_CATEGORIES[270] == "compiler";
+ assert WIDGET_IDS[271] == "wars";
+}
diff --git a/apps/website/public/widgets/nn-adam/card.png b/apps/website/public/widgets/nn-adam/card.png
new file mode 100644
index 0000000000..4694baeb17
Binary files /dev/null and b/apps/website/public/widgets/nn-adam/card.png differ
diff --git a/apps/website/public/widgets/nn-adam/index.html b/apps/website/public/widgets/nn-adam/index.html
new file mode 100644
index 0000000000..ba971d2d8d
--- /dev/null
+++ b/apps/website/public/widgets/nn-adam/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Adam: the optimizer most networks are trained with · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Adam: the optimizer most networks are trained with
+
Watch Adam carry a weight from 0 to the minimum at 3, then set three gradients of any size: the bias correction makes the first step lr times their sign, whatever their size.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-adam/logic.js b/apps/website/public/widgets/nn-adam/logic.js
new file mode 100644
index 0000000000..c404640df0
--- /dev/null
+++ b/apps/website/public/widgets/nn-adam/logic.js
@@ -0,0 +1,44 @@
+// Generated by scripts/t27-logic.mjs from adam.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 75f3d191b54ab786de3aa085164717813022eb8f403da12fe5b55c7bda8aa73f
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-adam/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABLAZgAX8BfWAIf39/f39/f38Bf2ACfX8BfWABfQF/YAJ9fQF9YAV9fX1/fwF9AwkIAAECAwQFAwIEBQFwAQEBBQMBABAGCQF/AUGAgMAACwdeBwZtZW1vcnkCAAt0MjdfZGVzY2VuZAAAC3QyN19hY2NlcHRzAAMJdDI3X3JhdGlvAAQOdDI3X2ZpcnN0X3N0ZXAABQ10MjdfaXNfZmluaXRlAAYIdDI3X3Bvd2kABwrlBwi+AQMBfwF9AX8jgICAgABBMGsiASSAgICAACABQQA2AhQgAUEANgIQIAFBADYCDEMAAAAAIQJBACEDAkADQCAAIANGDQEgAUEANgIsIAFC9/z++/OO85UyNwIkIAFCzZmz7uPMmbM/NwIcIAEgAkMAAEDAkiICIAKSOAIYIAFBDGpBASABQRhqQQEgAUEQaiABQRRqIANBAWoiAyABQRxqEIGAgIAAGiABKgIMIQIMAAsLIAFBMGokgICAgAAgAguxAgMFfQJ/BX0gByoCECEIIAcqAgwhCSAHKgIIIQogByoCBCELIAcqAgAhDEEAIQ0gAiEHAkACQANAIAMgDSIORg0BIA5BAWohDSAHKgIAIQ8gB0EEaiEHIA8gD1sgDyAPkyIPIA9bcQ0ADAILC0MAAIA/IAqTIRBDAACAPyALkyERQwAAgD8gCyAGEIKAgIAAkyESQwAAgD8gCiAGEIKAgIAAkyETA0AgAUUNASAEIAsgBCoCAJQgESACKgIAIAggACoCAJSSIg+UkjgCACAFIAogBSoCAJQgDyAQIA+UlJIiDzgCACAAIAAqAgAgDCAEKgIAIBKVlCAJIA8gE5WRkpWTOAIAIAVBBGohBSAEQQRqIQQgAEEEaiEAIAJBBGohAiABQX9qIQEMAAsLIA4gA08LKAEBfUMAAIA/IQICQANAIAFFDQEgAUF/aiEBIAAgApQhAgwACwsgAgueAQECfyOAgICAAEEwayIBJICAgIAAIAFBADYCLCABQvf8/vvzjvOVMjcCJCABQu+kjNTjzJmzPzcCHCABQQA2AhggAUEANgIUIAEgADgCECABQYCAgPwDNgIMIAFBDGpBASABQRBqQQEgAUEUaiABQRhqQQEgAUEcahCBgICAACECIAEqAgwhACABQTBqJICAgIAAIAIgAEMAAIA/XHELBwAgACABlQv+AQECfyOAgICAAEHQAGsiBSSAgICAACAFQTBqQQhqQQA2AgAgBUEgakEIakEANgIAIAVBADYCTCAFQvf8/vvzjvOVMjcCRCAFQu+kjNTjzJmzPzcCPCAFQgA3AzAgBUIANwMgIAUgAjgCHCAFIAE4AhggBSAAOAIUIAVBgICAggQ2AhAgBUKAgID8g4CAgMAANwIIIAVBCGpBAyAFQRRqQQMgBUEgaiAFQTBqQQEgBUE8ahCBgICAABogBUEgaiEGAkACQAJAIANBf2oOAgIAAQsgBUEwaiEGDAELIAVBCGohBgsgBiAEQQJ0aioCACECIAVB0ABqJICAgIAAIAILEgAgACAAWyAAIACTIgAgAFtxCwwAIAAgARCCgICAAAs=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function powi(b, t) {
+ return W.t27_powi(b, int(t, 0, 4294967295, "powi.t"));
+}
+function isFinite(x) {
+ return W.t27_is_finite(x) !== 0;
+}
+function firstStep(g0, g1, g2, which, index) {
+ return W.t27_first_step(g0, g1, g2, int(which, 0, 255, "first_step.which"), int(index, 0, 4294967295, "first_step.index"));
+}
+function ratio(a, b) {
+ return W.t27_ratio(a, b);
+}
+function accepts(g) {
+ return W.t27_accepts(g) !== 0;
+}
+function descend(steps) {
+ return W.t27_descend(int(steps, 0, 4294967295, "descend.steps"));
+}
+const __NOT_LOWERED__ = Object.freeze(["step", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ accepts,
+ descend,
+ firstStep,
+ isFinite,
+ powi,
+ ratio
+};
diff --git a/apps/website/public/widgets/nn-adam/logic.t27 b/apps/website/public/widgets/nn-adam/logic.t27
new file mode 100644
index 0000000000..df8bd6334d
--- /dev/null
+++ b/apps/website/public/widgets/nn-adam/logic.t27
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Adam adapts learning rate per parameter | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: step only checked its gradients with an undeclared `math::is_finite`
+// and updated nothing; the tests asserted `result != undefined`. step is now
+// Adam (Kingma and Ba, 2015) with bias correction. It refuses (returns false,
+// changes nothing) when a gradient is NaN or infinite, which is what the old body
+// validated.
+
+module Adam;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_BETA1 : f32 = 0.9;
+ const DEFAULT_BETA2 : f32 = 0.999;
+ const DEFAULT_EPSILON : f32 = 1e-8;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const AdamConfig = struct {
+ learning_rate : f32,
+ beta1 : f32,
+ beta2 : f32,
+ epsilon : f32,
+ weight_decay : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // b^t for a whole t >= 0
+ fn powi(b: f32, t: u32) -> f32 {
+ var r : f32 = 1.0;
+ var n : u32 = 0;
+ while (n < t) {
+ r = r * b;
+ n = n + 1;
+ }
+ return r;
+ }
+
+ // NaN is the one value not equal to itself; inf - inf is NaN
+ fn is_finite(x: f32) -> bool {
+ return x == x and (x - x) == (x - x);
+ }
+
+ // One Adam step number t (1-based) over params, with moment buffers m and v:
+ // g = grad + wd p
+ // m = b1 m + (1 - b1) g v = b2 v + (1 - b2) g^2
+ // m_hat = m / (1 - b1^t) v_hat = v / (1 - b2^t)
+ // p = p - lr m_hat / (sqrt(v_hat) + eps)
+ fn step(params: []f32, grads: []const f32, m: []f32, v: []f32, t: u32, config: AdamConfig) -> bool {
+ var i : usize = 0;
+ while (i < grads.len) {
+ if (!is_finite(grads[i])) {
+ return false;
+ }
+ i = i + 1;
+ }
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ i = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ m[i] = config.beta1 * m[i] + (1.0 - config.beta1) * g;
+ v[i] = config.beta2 * v[i] + (1.0 - config.beta2) * g * g;
+ const m_hat = m[i] / c1;
+ const v_hat = v[i] / c2;
+ params[i] = params[i] - config.learning_rate * m_hat / (sqrt(v_hat) + config.epsilon);
+ i = i + 1;
+ }
+ return true;
+ }
+
+ fn cfg(lr: f32) -> AdamConfig {
+ return AdamConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.0 };
+ }
+
+ // which: 0 = param, 1 = m, 2 = v after one step on params (1, 2, 3) with grads (g0, g1, g2)
+ fn first_step(g0: f32, g1: f32, g2: f32, which: u8, index: usize) -> f32 {
+ var params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [g0, g1, g2];
+ var m : [3]f32 = [0.0] ** 3;
+ var v : [3]f32 = [0.0] ** 3;
+ _ = step(¶ms, &grads, &m, &v, 1, cfg(0.001));
+ if (which == 1) {
+ return m[index];
+ }
+ if (which == 2) {
+ return v[index];
+ }
+ return params[index];
+ }
+
+ // a / b at run time, where IEEE gives NaN for 0 / 0 and inf for 1 / 0
+ fn ratio(a: f32, b: f32) -> f32 {
+ return a / b;
+ }
+
+ fn accepts(g: f32) -> bool {
+ var params : [1]f32 = [1.0];
+ const grads : [1]f32 = [g];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ return step(¶ms, &grads, &m, &v, 1, cfg(0.001)) and params[0] != 1.0;
+ }
+
+ // Minimise (x - 3)^2 from 0
+ fn descend(steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var m : [1]f32 = [0.0];
+ var v : [1]f32 = [0.0];
+ var t : u32 = 1;
+ while (t <= steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ _ = step(¶ms, &grads, &m, &v, t, cfg(0.1));
+ t = t + 1;
+ }
+ return params[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Bias correction makes the first step lr * sign(g), whatever |g| is
+ test step_basic_case
+ given p0 = first_step(0.1, -5.0, 0.0, 0, 0)
+ when p1 = first_step(0.1, -5.0, 0.0, 0, 1)
+ then @abs(p0 - 0.999) < 0.000001 and @abs(p1 - 2.001) < 0.000001 and first_step(0.1, -5.0, 0.0, 0, 2) == 3.0
+
+ // m = 0.1 g and v = 0.001 g^2 after the first step
+ test step_with_gradients
+ given m2 = first_step(0.1, 0.2, 0.3, 1, 2)
+ when v2 = first_step(0.1, 0.2, 0.3, 2, 2)
+ then @abs(m2 - 0.03) < 0.0000001 and @abs(v2 - 0.00009) < 0.0000001
+
+ // Finite gradients are applied; NaN and inf are refused
+ test step_validates_gradients
+ given ok = accepts(1.5) and accepts(-2.3) and accepts(4.7)
+ when refused = !accepts(ratio(0.0, 0.0)) and !accepts(ratio(1.0, 0.0))
+ then ok and refused
+
+ test step_descends_a_quadratic
+ given result = descend(500)
+ then @abs(result - 3.0) < 0.01
+
+ invariant adam_betas_are_below_one
+ assert DEFAULT_BETA1 < 1.0 and DEFAULT_BETA2 < 1.0 and DEFAULT_BETA1 < DEFAULT_BETA2 and DEFAULT_EPSILON > 0.0
diff --git a/apps/website/public/widgets/nn-adam/nn-adam.t27 b/apps/website/public/widgets/nn-adam/nn-adam.t27
new file mode 100644
index 0000000000..df20f42a52
--- /dev/null
+++ b/apps/website/public/widgets/nn-adam/nn-adam.t27
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-adam.t27 -- Adam: momentum, a running scale and the bias correction
+; Source of truth for public/widgets/nn-adam/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-adam/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/adam.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 18 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets the number of steps and three gradients from -50 to 50. The path is
+; descend(n) of the spec (lr 0.1 on (x - 3)^2 from 0); the bars and tiles are first_step(g0, g1, g2,
+; which, i) on weights (1, 2, 3) at lr 0.001.
+
+; WHAT IT DOES NOT CLAIM: the bowl is one weight, as in the SGD lesson, so the two paths can be
+; compared, not ranked.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_adam;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-adam";
+pub const TITLE : str = "Adam: the optimizer most networks are trained with";
+pub const DESCRIPTION : str = "Watch Adam carry a weight from 0 to the minimum at 3, then set three gradients of any size: the bias correction makes the first step lr times their sign, whatever their size.";
+pub const IMAGE_ALT : str = "Adam card: the green path rises from 0, passes the white dashed minimum at 3, peaks near 3.18 and settles at 3.00005 by step 200. Below, three bars of the first moment m read 0.20, -4.00 and 0.05 while the weights read 0.9990, 2.0010 and 2.9990. Tiles read x 3.00005 and step 1 at 0.1000.";
+pub const HOOK : str = "Gradients 2, -40 and 0.5 all move their weights by exactly 0.001 on step 1. Adam reaches 3.00005 in 200 steps at lr 0.1.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/adam.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 18 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS_HI : u8 = 200;
+pub const K_MINIMUM : u8 = 3;
+pub const K_Y_LO : u8 = 0;
+pub const K_Y_HI : u8 = 4;
+pub const K_Y_TICKS : [5]u8 = [0, 1, 2, 3, 4];
+pub const K_STEP_TICKS : [5]u8 = [0, 50, 100, 150, 200];
+pub const K_G_LO_C : i16 = -5000;
+pub const K_G_HI_C : u16 = 5000;
+pub const K_BAR_LO : i8 = -5;
+pub const K_BAR_HI : u8 = 5;
+pub const K_BAR_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_PRESET_STEPS : [4]u8 = [200, 10, 50, 200];
+pub const K_PRESET_G0_C : [4]i16 = [200, 100, -100, 5000];
+pub const K_PRESET_G1_C : [4]i16 = [-4000, 100, -100, 1];
+pub const K_PRESET_G2_C : [4]i16 = [50, 100, -100, -5000];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["200 steps; g 2, -40, 0.5", "10 steps; equal g", "50 steps; negative g", "200 steps; g 50, 0.01, -50"];
+pub const SAY_ASK : str = "Why is Adam the optimizer most networks are trained with?";
+pub const SAY_STEPS : str = "steps";
+pub const SAY_G : str = "gradient";
+pub const SAY_W : str = "w";
+pub const SAY_PLOT_LABEL : str = "The weight after every Adam step, from 0 toward the minimum at 3";
+pub const SAY_FIRST_LABEL : str = "Step 1 on weights 1, 2, 3: the first moment m and the weight after";
+pub const SAY_PATH : str = "Adam, lr 0.1";
+pub const SAY_MIN : str = "the minimum";
+pub const SAY_X_NOW : str = "x after the steps";
+pub const SAY_X_ONE : str = "after step 1";
+pub const SAY_W_AFTER : str = "weights after step 1";
+pub const SAY_V : str = "second moment v";
+pub const SAY_NOTE : str = "m and v start at zero, so early on they are too small; dividing by 1 - beta^t corrects them. On step 1 that makes m / sqrt(v) exactly the sign of the gradient.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/adam.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_gradients_are_symmetric {
+ assert K_G_LO_C + K_G_HI_C == 0;
+ assert K_PRESET_G0_C[3] == K_G_HI_C;
+ assert K_PRESET_G2_C[3] == K_G_LO_C;
+}
+
+test the_minimum_is_inside_the_axis {
+ assert K_MINIMUM == 3;
+ assert K_Y_HI > K_MINIMUM;
+ assert K_PRESET_STEPS[0] == K_STEPS_HI;
+}
diff --git a/apps/website/public/widgets/nn-adam/tool.js b/apps/website/public/widgets/nn-adam/tool.js
new file mode 100644
index 0000000000..8ce8c40385
--- /dev/null
+++ b/apps/website/public/widgets/nn-adam/tool.js
@@ -0,0 +1,44 @@
+// nn-adam/tool.js -- lesson 18 of neural-networks: the optimizer most networks are trained with.
+// Adam keeps a running mean of the gradient (m) and of its square (v), corrects both for starting at
+// zero, and steps by m / sqrt(v): so its first step is lr times the sign of the gradient, whatever
+// its size. Every point is descend and every bar first_step of gHashTag/t27
+// specs/ml/optimizer/adam.t27, compiled to wasm (./logic.js). Words are SAY_ and numbers K_ constants
+// of specs/widgets/nn-adam.t27.
+import { descend, firstStep } from './logic.js'
+import { COLORS, K, W, ask, bars, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const NMAX = K('K_STEPS_HI', 200)
+const MIN = K('K_MINIMUM', 3)
+const G0 = centi(K('K_G_LO_C', -5000)), G1 = centi(K('K_G_HI_C', 5000))
+const grad = (key, i) => ({ key, label: `${say('SAY_G')} ${i}`, min: G0, max: G1, step: 0.5, fmt: (v) => fmt(v, 1) })
+const ctl = controls([
+ { key: 'n', label: say('SAY_STEPS'), min: 1, max: NMAX },
+ grad('g0', 0), grad('g1', 1), grad('g2', 2),
+], { names: W.SAY_PRESET_NAMES, values: { n: K('K_PRESET_STEPS', []), g0: K('K_PRESET_G0_C', []).map(centi), g1: K('K_PRESET_G1_C', []).map(centi), g2: K('K_PRESET_G2_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const first = canvas('nn-bars', say('SAY_FIRST_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_PATH'), COLORS[0]], [say('SAY_MIN'), '#ffffff', true]]), el('p', { class: 'dk-label' }, say('SAY_FIRST_LABEL')), first, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 320, x0: 0, x1: NMAX, y0: K('K_Y_LO', 0), y1: K('K_Y_HI', 4), xticks: K('K_STEP_TICKS', []), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_STEPS') }
+ const p = plot(svg, o)
+ svg.append(svgEl('line', { x1: p.left, x2: 640 - p.right, y1: p.clampY(MIN), y2: p.clampY(MIN), stroke: '#fff', 'stroke-dasharray': '6 6', opacity: 0.7 }))
+ p.line([...Array(NMAX + 1)].map((_, k) => [k, descend(k)]), COLORS[0], 3)
+ cursor(svg, p, s.n, [[descend(s.n), COLORS[1]]])
+ dragX(svg, p, o, (n) => ctl.set('n', Math.round(n)))
+ // Bars: the first moment m, which grows with the gradient; under each, the weight after the step,
+ // which moves by the same lr whatever the gradient was.
+ const m = [0, 1, 2].map((i) => firstStep(s.g0, s.g1, s.g2, 1, i))
+ bars(first, { values: m, labels: [0, 1, 2].map((i) => `${say('SAY_W')} ${i} = ${fmt(firstStep(s.g0, s.g1, s.g2, 0, i), 4)}`), colors: m.map((v) => (v < 0 ? COLORS[3] : COLORS[0])), lo: K('K_BAR_LO', -5), hi: K('K_BAR_HI', 5), h: 230, ticks: K('K_BAR_TICKS', []), fmt: (v) => `m ${fmt(v, 2)}` })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_X_NOW'), value: fmt(descend(s.n), 5), tone: 0 },
+ { label: say('SAY_X_ONE'), value: fmt(descend(1), 4), tone: 1 },
+ { label: say('SAY_W_AFTER'), value: [0, 1, 2].map((i) => fmt(firstStep(s.g0, s.g1, s.g2, 0, i), 4)).join(' '), tone: 2 },
+ { label: say('SAY_V'), value: [0, 1, 2].map((i) => fmt(firstStep(s.g0, s.g1, s.g2, 2, i), 4)).join(' '), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-attention/card.png b/apps/website/public/widgets/nn-attention/card.png
new file mode 100644
index 0000000000..0c6baf600b
Binary files /dev/null and b/apps/website/public/widgets/nn-attention/card.png differ
diff --git a/apps/website/public/widgets/nn-attention/index.html b/apps/website/public/widgets/nn-attention/index.html
new file mode 100644
index 0000000000..b66fd3e65e
--- /dev/null
+++ b/apps/website/public/widgets/nn-attention/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Attention: how a word decides which other words to read · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Attention: how a word decides which other words to read
+
Two heads of attention over three positions: each row is how much one query reads each key. Switch on the causal mask and a position can no longer read the future.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-attention/logic.js b/apps/website/public/widgets/nn-attention/logic.js
new file mode 100644
index 0000000000..6c908634c9
--- /dev/null
+++ b/apps/website/public/widgets/nn-attention/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from attention_mechanism.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: f3f3c3bf7bd4d4d79173dc34896aa185857d5abc3e8684c3bfcd949547c9b591
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-attention/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABOQlgAAF/YAJ/fwF9YAN/f30BfWADf39/AGAGf39/f39/AGADf39/AX1gAn19AX1gAX0BfWACfX8BfQMMCwABAgMEAgIFBgcIBAUBcAEBAQUDAQARBgkBfwFBgIDAAAsHTwYGbWVtb3J5AgAPdDI3X3ZhbGlkX2lucHV0AAAJdDI3X3Nsb3BlAAEIdDI3X2dyYWQABQh0MjdfbG9zcwAGDnQyN19mb3J3YXJkX2F0AAcK2BYLBABBAAsnACAAIAFDbxKDOhCCgICAACAAIAFDbxKDuhCCgICAAJNDbxIDO5ULjwIBAn8jgICAgABBoAJrIgMkgICAgAAgA0EEaiADQTRqIANB5ABqEIOAgIAAIANB5ABqIQQCQAJAAkACQCAAQf8BcQ4DAAECAwsgA0EEaiEEDAELIANBNGohBAsgBCABQQJ0aiIEIAIgBCoCAJI4AgALIANBADYClAIgA0KEgICAIDcCjAIgA0EAOgCcAiADQQA6AJgCIANBBGogA0E0aiADQeQAaiADQZQBaiADQcQBaiADQYwCahCEgICAAEMAAAAAIQIgA0GUAWohAEEBIQQCQANAIARBDUYNASACIASzQwAAIEGVIAAqAgCUkiECIABBBGohACAEQQFqIQQMAAsLIANBoAJqJICAgIAAIAILdAMBfwF9AX9BASEDAkADQCADQQ1GDQEgACADs0MAACBBlSIEOAIAIAEgA0F/aiIFQQJ2QQFqs0MAAIA+lCAFQQFxs0MAAEDAlZI4AgAgAiAEOAIAIAJBBGohAiABQQRqIQEgAEEEaiEAIANBAWohAwwACwsLzAQGBH8BfQx/An0CfwF9QQwgBSgCACIGbiEHQwAAgD8gBiAFKAIEIghuIgmzkZUhCkEAIQsgBS0ADEEBcSEMAkADQCALIAhGDQEgCyAHbCENIAMgCyAJbEECdCIFaiEOIAIgBWohDyABIAVqIRAgACAFaiERQQAhEgJAA0AgEiAHTw0BIBEgEiAGbEECdCITaiEUIAQgEiANaiAHbEECdGohFUEAIRZDyvJJ8SEXAkADQAJAIBYgB0kNAEMAAAAAIRhBACEFDAILAkACQCAWIBJNDQBDyvJJ8SEYIAwNAQsgECAWIAZsQQJ0aiEZQQAhBUMAAAAAIRgCQANAIAUgCU8NASAYIBQgBUECdCIaaioCACAZIBpqKgIAlJIhGCAFQQFqIQUMAAsLIAogGJQhGAsgFSAWQQJ0aiAYOAIAIBZBAWohFiAXIBgQiICAgAAhFwwACwsCQANAIAUgB08NASAVIAVBAnRqIRogGiAaKgIAIBeTEImAgIAAIhs4AgAgBUEBaiEFIBggG5IhGAwACwtBACEFAkADQAJAIAUgB0kNACAOIBNqIRZBACEZA0AgGSAJTw0DIA8gGUECdCIUaiEaQwAAAAAhGEEAIQUCQANAIAUgB08NASAYIBUgBUECdGoqAgAgGiAFIAZsQQJ0aioCAJSSIRggBUEBaiEFDAALCyAWIBRqIBg4AgAgGUEBaiEZDAALCyAVIAVBAnRqIhogGioCACAYlTgCACAFQQFqIQUMAAsLIBJBAWohEgwACwsgC0EBaiELDAALCwuiBwQVfwF9An8CfSOAgICAAEHgA2siAySAgICAACADQQhqIANBOGogA0HoAGoQg4CAgAAgAyACOAKYAiADQoSAgIAgNwKQAiADQQA6AJwCIANBADoAyAEgA0EIaiADQThqIANB6ABqIANBmAFqIANByAFqIANBkAJqEISAgIAAIANBoAJqIQRBASEFAkADQAJAIAVBDUcNAEEAIQUDQAJAIAVBMEcNAEEAIQYgA0HoAGohByADQbADaiEIIANBoAJqIQkgA0E4aiEKIANBgANqIQsgA0HQAmohDCADQQhqIQ0DQCAGQQJGDQUgBkEDbCEOIANB6ABqIAZBA3QiBWohDyADQaACaiAFaiEQQQAhESANIRIgDCETIAkhFAJAA0AgEUEDRg0BIBAgEUEEdGohFSADQcgBaiARIA5qQQxsaiEWQQAhF0MAAAAAIRggCCEZIAchGgJAA0ACQCAXQQNHDQAgFUEEaioCACEbIBUqAgAhHEEAIRcgCyEZIAohGgwCCyAWIBdBAnRqKgIAIRxDAAAAACECQQAhBQJAA0AgBUEIRg0BIBkgBWoiBCAcIBQgBWoqAgAiG5QgBCoCAJI4AgAgAiAbIBogBWoqAgCUkiECIAVBBGohBQwACwsgGUEQaiEZIBpBEGohGiAXQQFqIRcgGCAcIAKUkiEYDAALCwNAAkAgF0EDRw0AIBJBEGohEiATQRBqIRMgFEEQaiEUIBFBAWohEQwCCyAWIBdBAnRqKgIAIBwgDyAXQQR0aiIFKgIAlEMAAAAAkiAbIAVBBGoqAgCUkiAYk5RD8wQ1P5QhAkEAIQUCQANAIAVBCEYNASATIAVqIgQgBCoCACACIBogBWoqAgCUkjgCACAZIAVqIgQgBCoCACACIBIgBWoqAgCUkjgCACAFQQRqIQUMAAsLIBlBEGohGSAaQRBqIRogF0EBaiEXDAALCwsgDUEIaiENIAxBCGohDCALQQhqIQsgCkEIaiEKIAlBCGohCSAIQQhqIQggB0EIaiEHIAZBAWohBgwACwsgA0GAA2ogBWpBADYCACADQdACaiAFakEANgIAIANBsANqIAVqQQA2AgAgBUEEaiEFDAALCyAEIAWzQwAAIEGVOAIAIARBBGohBCAFQQFqIQUMAAsLIANB0AJqIQUCQAJAAkAgAA4CAgABCyADQYADaiEFDAELIANBsANqIQULIAUgAUECdGoqAgAhAiADQeADaiSAgICAACACCw4AIAAgASACEIKAgIAAC90BAgF/AX0jgICAgABBoAJrIgMkgICAgAAgA0EEaiADQTRqIANB5ABqEIOAgIAAAkBBMEUNACADQZQBakGAgMCAAEEw/AoAAAsCQEHIAEUNACADQcQBakGwgMCAAEHIAPwKAAALIANBADYClAIgA0KEgICAIDcCjAIgAyAAQQFxIgA6AJgCIAMgADoAnAIgA0EEaiADQTRqIANB5ABqIANBlAFqIANBxAFqIANBjAJqEISAgIAAIANBxAFqIANBlAFqIAEbIAJBAnRqKgIAIQQgA0GgAmokgICAgAAgBAtFAQF/AkAgACAAXA0AAkAgASABWw0AIAAPCwJAIAG8IAC8IgJzQX9KDQAgASAAIAJBAEgbDwsgASAAIAAgAV0bIQELIAEL1AICA38DfQJAAkAgACAAXA0AIAC8IgFBH3YhAgJAAkACQAJAAkACQAJAIAFB/////wdxIgNBz9i6lQRLDQAgA0GY5MX1A00NASADQZKrlPwDTQ0CDAQLIANBgICA/AdLDQYCQCABQQBIDQAgA0GY5MWVBEkNACAAQwAAAH+UDwsgAUF/Sg0DQwAAAAAhBCADQbTjv5YESw0HDAMLIANBgICAyANNDQFBACEDQwAAAAAhBSAAIQQMBAsgAkEBcyACayEDDAILIABDAACAP5IPCyAAQzuquD+UIAJBAnQqAviAwIAAkvwAIQMLIAAgA7IiBEMAcjG/lJIiACAEQ46+vzWUIgWTIQQLIAAgBCAEIAQgBJQiBiAGQxVSNbuUQ4+qKj6SlJMiBpRDAAAAQCAGk5UgBZOSQwAAgD+SIQQgA0UNASAEIAMQioCAgAAPCyAAIQQLIAQLpgIBBH8CQAJAIAAgAFwNACAAvCICQf////8HcUGAgID8B08NACACQYCAgIB4cSEDQQggAkEBdCIEZ2sgBEEYdiAEQYCAgAhJGyEEAkAgAUEASA0AAkAgAUH+ASAEa0wNAEMAAIB/IACYDwsCQCAEIAFqIgVBAEoNACACIAF0IANyvg8LAkAgBEEASg0AIAJBASAEa3RB////A3EgBUEXdHIgA3K+DwsgAUEXdCACar4PCyABQQAgBGtKDQECQCABQWkgBGtODQAgA74PC0GAgIAEQQAgBEEAShsgAkH///8DcXJBACABayIDQQEgASAEamsiASADIAFIG0F/aiIBdkEBakEBdiACaCABQR9xRkF/c3G+IACYIQALIAAPCyABQRd0IAJqvgsLigEBAEGAgMAAC4ABAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAAPwAAAL8=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function forwardAt(causal, which, index) {
+ return W.t27_forward_at(causal ? 1 : 0, int(which, 0, 255, "forward_at.which"), int(index, 0, 4294967295, "forward_at.index"));
+}
+function loss(which, index, delta) {
+ return W.t27_loss(int(which, 0, 255, "loss.which"), int(index, 0, 4294967295, "loss.index"), delta);
+}
+function grad(which, index, dropout) {
+ return W.t27_grad(int(which, 0, 255, "grad.which"), int(index, 0, 4294967295, "grad.index"), dropout);
+}
+function slope(which, index) {
+ return W.t27_slope(int(which, 0, 255, "slope.which"), int(index, 0, 4294967295, "slope.index"));
+}
+function validInput() {
+ return W.t27_valid_input() >>> 0;
+}
+const __NOT_LOWERED__ = Object.freeze(["seq_of", "scale_of", "forward", "backward", "cfg", "fixture"]);
+export {
+ __NOT_LOWERED__,
+ forwardAt,
+ grad,
+ loss,
+ slope,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-attention/logic.t27 b/apps/website/public/widgets/nn-attention/logic.t27
new file mode 100644
index 0000000000..48dce23e6c
--- /dev/null
+++ b/apps/website/public/widgets/nn-attention/logic.t27
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Memory-efficient attention for long sequences | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: redeclared `var i` in one scope four times, called `@infinity()` and
+// `@random()` (not Zig), and indexed every row as head_dim wide inside d_model-wide
+// rows. backward also skipped the softmax Jacobian: it pushed dL/dweights straight
+// into the scores. It is now multi-head scaled dot-product attention over seq x
+// d_model rows. Head h reads columns h*hd .. (h+1)*hd, the weights are
+// heads x seq x seq, and causal masking is optional. backward is the exact
+// gradient, checked against finite differences. Dropout is carried in the config
+// and not applied: this is the inference computation.
+
+module Attention;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const NEG_INF : f32 = -1.0e30;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const AttentionConfig = struct {
+ d_model : u32,
+ num_heads : u32,
+ dropout : f32,
+ causal : bool,
+ };
+
+ pub const AttentionOutput = struct {
+ output : []f32,
+ weights : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn seq_of(query: []const f32, config: AttentionConfig) -> usize {
+ return query.len / config.d_model;
+ }
+
+ fn scale_of(config: AttentionConfig) -> f32 {
+ return 1.0 / sqrt(@as(f32, @floatFromInt(config.d_model / config.num_heads)));
+ }
+
+ fn forward(query: []const f32, key: []const f32, value: []const f32, output: []f32, weights: []f32, config: AttentionConfig) -> void {
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const row = weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var max_s : f32 = NEG_INF;
+ var j : usize = 0;
+ while (j < n) {
+ var score : f32 = NEG_INF;
+ if (!(config.causal and j > i)) {
+ score = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ score = score + query[i * d + h * hd + k] * key[j * d + h * hd + k];
+ k = k + 1;
+ }
+ score = score * scale;
+ }
+ row[j] = score;
+ max_s = @max(max_s, score);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ j = 0;
+ while (j < n) {
+ row[j] = row[j] / sum;
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < hd) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < n) {
+ acc = acc + row[j] * value[j * d + h * hd + k];
+ j = j + 1;
+ }
+ output[i * d + h * hd + k] = acc;
+ k = k + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ // dV_j += P_ij dO_i; dP_ij = dO_i . V_j; dS_ij = P_ij (dP_ij - sum_k P_ik dP_ik);
+ // dQ_i += scale dS_ij K_j; dK_j += scale dS_ij Q_i (per head; masked P_ij are 0)
+ fn backward(grad_output: []const f32, query: []const f32, key: []const f32, value: []const f32, attn_weights: []f32, grad_query: []f32, grad_key: []f32, grad_value: []f32, config: AttentionConfig) -> void {
+ const n = seq_of(query, config);
+ const d : usize = config.d_model;
+ const hd : usize = config.d_model / config.num_heads;
+ const scale = scale_of(config);
+ var x : usize = 0;
+ while (x < n * d) {
+ grad_query[x] = 0.0;
+ grad_key[x] = 0.0;
+ grad_value[x] = 0.0;
+ x = x + 1;
+ }
+ var h : usize = 0;
+ while (h < config.num_heads) {
+ var i : usize = 0;
+ while (i < n) {
+ const p = attn_weights[(h * n + i) * n .. (h * n + i + 1) * n];
+ var row_dot : f32 = 0.0;
+ var j : usize = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ grad_value[j * d + h * hd + k] = grad_value[j * d + h * hd + k] + p[j] * grad_output[i * d + h * hd + k];
+ k = k + 1;
+ }
+ row_dot = row_dot + p[j] * dp;
+ j = j + 1;
+ }
+ j = 0;
+ while (j < n) {
+ var dp : f32 = 0.0;
+ var k : usize = 0;
+ while (k < hd) {
+ dp = dp + grad_output[i * d + h * hd + k] * value[j * d + h * hd + k];
+ k = k + 1;
+ }
+ const ds = p[j] * (dp - row_dot) * scale;
+ k = 0;
+ while (k < hd) {
+ grad_query[i * d + h * hd + k] = grad_query[i * d + h * hd + k] + ds * key[j * d + h * hd + k];
+ grad_key[j * d + h * hd + k] = grad_key[j * d + h * hd + k] + ds * query[i * d + h * hd + k];
+ k = k + 1;
+ }
+ j = j + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ fn cfg(causal: bool, dropout: f32) -> AttentionConfig {
+ return AttentionConfig{ .d_model = 4, .num_heads = 2, .dropout = dropout, .causal = causal };
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads (columns 0-1 and 2-3); q / k / v scaled down so the
+ // softmax is not saturated. which: 0 = output, 1 = weights.
+ fn fixture(q: []f32, k: []f32, v: []f32) -> void {
+ var x : usize = 0;
+ while (x < 12) {
+ q[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ k[x] = @as(f32, @floatFromInt(x / 4 + 1)) / 4.0 - @as(f32, @floatFromInt(x % 2)) / 3.0;
+ v[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
+ }
+ }
+
+ fn forward_at(causal: bool, which: u8, index: usize) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = [9.0] ** 12;
+ var w : [18]f32 = [9.0] ** 18;
+ forward(&q, &k, &v, &out, &w, cfg(causal, 0.0));
+ if (which == 0) {
+ return out[index];
+ }
+ return w[index];
+ }
+
+ // L = sum_x g_x O_x with g_x = (x + 1) / 10, one entry of q, k or v moved by delta
+ // (which: 0 = q, 1 = k, 2 = v)
+ fn loss(which: u8, index: usize, delta: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ if (which == 0) {
+ q[index] = q[index] + delta;
+ }
+ if (which == 1) {
+ k[index] = k[index] + delta;
+ }
+ if (which == 2) {
+ v[index] = v[index] + delta;
+ }
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, 0.0));
+ var total : f32 = 0.0;
+ var x : usize = 0;
+ while (x < 12) {
+ total = total + @as(f32, @floatFromInt(x + 1)) / 10.0 * out[x];
+ x = x + 1;
+ }
+ return total;
+ }
+
+ fn grad(which: u8, index: usize, dropout: f32) -> f32 {
+ var q : [12]f32 = undefined;
+ var k : [12]f32 = undefined;
+ var v : [12]f32 = undefined;
+ fixture(&q, &k, &v);
+ var out : [12]f32 = undefined;
+ var w : [18]f32 = undefined;
+ forward(&q, &k, &v, &out, &w, cfg(false, dropout));
+ var g : [12]f32 = undefined;
+ var x : usize = 0;
+ while (x < 12) {
+ g[x] = @as(f32, @floatFromInt(x + 1)) / 10.0;
+ x = x + 1;
+ }
+ var gq : [12]f32 = undefined;
+ var gk : [12]f32 = undefined;
+ var gv : [12]f32 = undefined;
+ backward(&g, &q, &k, &v, &w, &gq, &gk, &gv, cfg(false, dropout));
+ if (which == 0) {
+ return gq[index];
+ }
+ if (which == 1) {
+ return gk[index];
+ }
+ return gv[index];
+ }
+
+ fn slope(which: u8, index: usize) -> f32 {
+ return (loss(which, index, 0.001) - loss(which, index, -0.001)) / 0.002;
+ }
+
+ fn valid_input() -> u32 {
+ return cfg(false, 0.0).d_model % cfg(false, 0.0).num_heads;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Each weight row is a distribution, and each output is a weighted mean of the values
+ // (so it lies between the smallest and largest value in its column)
+ test forward_basic_case {
+ const row_sum = forward_at(false, 1, 0) + forward_at(false, 1, 1) + forward_at(false, 1, 2);
+ assert(@abs(row_sum - 1.0) < 0.000001);
+ assert(forward_at(false, 0, 0) + forward_at(false, 0, 1) + forward_at(false, 0, 2) + forward_at(false, 0, 3) > 0.0); // computed a meaningful output
+ assert(forward_at(false, 0, 0) > 0.1 and forward_at(false, 0, 0) < 0.9);
+ }
+
+ // dQ, dK and dV against central differences of L = g . O
+ test backward_basic_case {
+ assert(@abs(grad(0, 0, 0.0) - slope(0, 0)) < 0.001);
+ assert(@abs(grad(0, 7, 0.0) - slope(0, 7)) < 0.001);
+ assert(@abs(grad(1, 5, 0.0) - slope(1, 5)) < 0.001);
+ assert(@abs(grad(2, 10, 0.0) - slope(2, 10)) < 0.001);
+ assert(grad(0, 0, 0.0) + grad(0, 1, 0.0) + grad(0, 2, 0.0) + grad(0, 3, 0.0) != 0.0); // computed gradients
+ }
+
+ // Position 0 attends only to itself, position 1 to positions 0 and 1
+ test forward_causal_masking {
+ assert(forward_at(true, 1, 0) == 1.0);
+ assert(forward_at(true, 1, 1) == 0.0 and forward_at(true, 1, 2) == 0.0);
+ assert(forward_at(true, 1, 1 * 3 + 2) < 0.999);
+ assert(forward_at(true, 1, 1 * 3 + 2) == 0.0);
+ assert(forward_at(true, 0, 0) == 0.1);
+ }
+
+ // Dropout is not applied here (inference), so it leaves the gradients unchanged
+ test backward_dropout {
+ assert(grad(2, 3, 0.5) == grad(2, 3, 0.0));
+ assert(grad(0, 11, 0.5) == grad(0, 11, 0.0));
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // d_model % num_heads == 0
+ invariant attention_constraint_0
+ assert valid_input() == 0
+
+ // 0 <= dropout < 1
+ invariant attention_constraint_1
+ assert cfg(false, 0.5).dropout >= 0.0 and cfg(false, 0.5).dropout < 1.0
+
+ // query.shape == key.shape == value.shape: seq is read from the query
+ invariant attention_constraint_2
+ assert cfg(false, 0.0).d_model == 4
diff --git a/apps/website/public/widgets/nn-attention/nn-attention.t27 b/apps/website/public/widgets/nn-attention/nn-attention.t27
new file mode 100644
index 0000000000..32f3a9f038
--- /dev/null
+++ b/apps/website/public/widgets/nn-attention/nn-attention.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-attention.t27 -- scaled dot-product attention with two heads and a causal mask
+; Source of truth for public/widgets/nn-attention/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-attention/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/recurrent/attention_mechanism.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 24 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader toggles the causal mask and taps a query row. Every weight is
+; forward_at(causal, 1, i) and every output forward_at(causal, 0, i) of the spec: 3 positions,
+; d_model 4, 2 heads, its own q, k and v.
+
+; WHAT IT DOES NOT CLAIM: q, k and v are the spec's fixture, not learned projections of real tokens.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_attention;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-attention";
+pub const TITLE : str = "Attention: how a word decides which other words to read";
+pub const DESCRIPTION : str = "Two heads of attention over three positions: each row is how much one query reads each key. Switch on the causal mask and a position can no longer read the future.";
+pub const IMAGE_ALT : str = "Attention card, causal: two 3 x 3 grids in blue to white. Head 0 reads 1.000 / 0.452 0.548 / 0.230 0.321 0.449 with the upper triangle at 0; head 1 1.000 / 0.434 0.566 / 0.210 0.316 0.474. Row 2 is outlined. Below, a 3 x 4 output grid; position 2 outputs 0.59 0.69 0.81 0.91.";
+pub const HOOK : str = "Causal: position 0 reads only itself (1.000), position 1 splits 0.45 / 0.55, position 2 spreads 0.23, 0.32, 0.45. Every row is a softmax.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/recurrent/attention_mechanism.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 24 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SEQ : u8 = 3;
+pub const K_D_MODEL : u8 = 4;
+pub const K_HEADS : u8 = 2;
+pub const K_PRESET_CAUSAL : [3]u8 = [1, 0, 1];
+pub const K_PRESET_Q : [3]u8 = [2, 2, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["Causal, position 2", "Full, position 2", "Causal, position 1"];
+pub const SAY_ASK : str = "How does a word decide which other words to read?";
+pub const SAY_CAUSAL : str = "causal mask";
+pub const SAY_HEAD : str = "head";
+pub const SAY_POS : str = "pos";
+pub const SAY_DIM : str = "dim";
+pub const SAY_ROWS : str = "rows are queries, columns keys";
+pub const SAY_TAP : str = "Tap a row to follow one query.";
+pub const SAY_OUT_LABEL : str = "The output: each position's weighted average of the values";
+pub const SAY_MASK : str = "mask";
+pub const SAY_MASK_ON : str = "no reading ahead";
+pub const SAY_MASK_OFF : str = "every key";
+pub const SAY_OUT_ROW : str = "output of {0}";
+pub const SAY_NOTE : str = "Each weight is softmax(q . k / sqrt(d)) over the keys of its row; the output is the values averaged by those weights. Two heads look at different halves of the vector.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/recurrent/attention_mechanism.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test two_heads_split_the_model {
+ assert K_HEADS * 2 == K_D_MODEL;
+ assert K_SEQ == 3;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_Q[0] < K_SEQ;
+ assert K_PRESET_CAUSAL[0] == 1;
+ assert K_PRESET_CAUSAL[1] == 0;
+}
diff --git a/apps/website/public/widgets/nn-attention/tool.js b/apps/website/public/widgets/nn-attention/tool.js
new file mode 100644
index 0000000000..8460014cda
--- /dev/null
+++ b/apps/website/public/widgets/nn-attention/tool.js
@@ -0,0 +1,37 @@
+// nn-attention/tool.js -- lesson 24 of neural-networks: how a word decides which other words to read.
+// Attention scores every query against every key, turns each row of scores into weights with a
+// softmax and averages the values by them; a causal mask stops a position from reading the future.
+// Every weight and output is forward_at of gHashTag/t27 specs/ml/recurrent/attention_mechanism.t27,
+// compiled to wasm (./logic.js): 3 positions, d_model 4, 2 heads, the spec's own q, k and v. Words
+// are SAY_ and numbers K_ constants of specs/widgets/nn-attention.t27.
+import { forwardAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fill, fmt, heat, heatTone, root, say, source, tiles, tone } from '../nn-kit.js'
+
+const N = K('K_SEQ', 3), D = K('K_D_MODEL', 4), H = K('K_HEADS', 2)
+const ctl = controls([
+ { key: 'causal', label: say('SAY_CAUSAL'), kind: 'toggle' },
+ { key: 'q', kind: 'hidden', min: 0, max: 2 },
+], { names: W.SAY_PRESET_NAMES, values: { causal: K('K_PRESET_CAUSAL', []), q: K('K_PRESET_Q', []) } }, draw)
+
+const heads = [...Array(H)].map((_, h) => canvas('nn-heat', `${say('SAY_HEAD')} ${h}`))
+const out = canvas('nn-heat', say('SAY_OUT_LABEL'))
+const stat = el('div')
+const pos = [...Array(N)].map((_, i) => `${say('SAY_POS')} ${i}`)
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('p', { class: 'dk-note' }, say('SAY_TAP')),
+ el('div', { class: 'nn-two' }, ...heads.map((svg, h) => el('div', {}, el('p', { class: 'dk-label' }, `${say('SAY_HEAD')} ${h}: ${say('SAY_ROWS')}`), svg))),
+ el('p', { class: 'dk-label' }, say('SAY_OUT_LABEL')), out, stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const c = !!s.causal
+ const w = (h, i, j) => forwardAt(c, 1, (h * N + i) * N + j)
+ heads.forEach((svg, h) => heat(svg, { rows: N, cols: N, value: (i, j) => w(h, i, j), color: (v) => heatTone(v), rowLabels: pos, colLabels: pos, labelWidth: 70, w: 400, cell: 100, fmt: (v) => fmt(v, 3), mark: (i) => (i === s.q ? 'on' : ''), onTap: (i) => ctl.set('q', i) }))
+ heat(out, { rows: N, cols: D, value: (i, k) => forwardAt(c, 0, i * D + k), color: (v) => tone(v, 1.2), rowLabels: pos, colLabels: [...Array(D)].map((_, k) => `${say('SAY_DIM')} ${k}`), labelWidth: 70, w: 640, cell: 110, fmt: (v) => fmt(v, 3), mark: (i) => (i === s.q ? 'on' : ''), onTap: (i) => ctl.set('q', i) })
+ stat.replaceChildren(tiles([
+ { label: `${say('SAY_HEAD')} 0, ${pos[s.q]}`, value: [0, 1, 2].map((j) => fmt(w(0, s.q, j), 3)).join(' '), tone: 2 },
+ { label: `${say('SAY_HEAD')} 1, ${pos[s.q]}`, value: [0, 1, 2].map((j) => fmt(w(1, s.q, j), 3)).join(' '), tone: 3 },
+ { label: say('SAY_MASK'), value: c ? say('SAY_MASK_ON') : say('SAY_MASK_OFF'), tone: c ? 1 : 5 },
+ { label: fill(say('SAY_OUT_ROW'), pos[s.q]), value: [0, 1, 2, 3].map((k) => fmt(forwardAt(c, 0, s.q * D + k), 2)).join(' '), tone: 0 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-backprop/card.png b/apps/website/public/widgets/nn-backprop/card.png
new file mode 100644
index 0000000000..20586d19b8
Binary files /dev/null and b/apps/website/public/widgets/nn-backprop/card.png differ
diff --git a/apps/website/public/widgets/nn-backprop/index.html b/apps/website/public/widgets/nn-backprop/index.html
new file mode 100644
index 0000000000..25b8bad9a4
--- /dev/null
+++ b/apps/website/public/widgets/nn-backprop/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Backpropagation: how blame flows back through a layer · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Backpropagation: how blame flows back through a layer
+
Set how much the loss wants each of two outputs to move: every weight gets that times its input, every input gets W transposed times it. The edges and nodes show the numbers.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-backprop/logic.js b/apps/website/public/widgets/nn-backprop/logic.js
new file mode 100644
index 0000000000..247b5bdd4b
--- /dev/null
+++ b/apps/website/public/widgets/nn-backprop/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from dense_layer.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: c756c98dad128be955c1a56ed60b9ea25d6108c91f92065f69f5a8d76ce0422b
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-backprop/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABNAdgAAF9YAR9fX1/AX1gBX9/f39/AGAAAX9gB39/f39/f38AYAR9fX9/AX1gBX19fX19AX0DCQgAAQIDBAMFBgQFAXABAQEFAwEAEQYJAX8BQYCAwAALB3kGBm1lbW9yeQIAD3QyN192YWxpZF9pbnB1dAAAHHQyN19lbXB0eV9iYWNrd2FyZF91bnRvdWNoZWQAAxt0MjdfZW1wdHlfZm9yd2FyZF91bnRvdWNoZWQABQ90MjdfYmFja3dhcmRfYXQABgt0MjdfbG9zc19hdAAHCsgICBkAQwAAgD9DAAAAQEMAAEBAQQAQgYCAgAALgQEBAX8jgICAgABBMGsiBCSAgICAACAEIAI4AhQgBCABOAIQIAQgADgCDCAEQgA3AxggBEKDgICAIDcCICAEQQE7ASggBEEMakGAgMCAAEGYgMCAACAEQSBqIARBGGoQgoCAgAAgBEEYaiADQQJ0aioCACECIARBMGokgICAgAAgAgvTAQMGfwF9An8gAygCACIFQQJ0IQYgAygCBCEHQQAhCCADLQAIQQFxIQkgAy0ACUH/AXFBAUchCgNAAkACQAJAIAggB0YNAEMAAAAAIQsgCQ0BDAILDwsgAiAIQQJ0aioCACELCyAAIQwgASENIAUhAwJAA0AgA0UNASADQX9qIQMgCyANKgIAIAwqAgCUkiELIAxBBGohDCANQQRqIQ0MAAsLIAQgCEECdGogCyALQwAAAAAgC0MAAAAAXhsgChs4AgAgASAGaiEBIAhBAWohCAwACwtdAgF/AX0jgICAgABBEGsiACSAgICAACAAQYCAgIcENgIMQQRBBEEEQaCAwIAAIABBDGogAEEMaiAAQQxqEISAgIAAIAAqAgwhASAAQRBqJICAgIAAIAFDAADgQFsL6QEBCX8gAygCBCEHIAQhCCADKAIAIgkhAwJAA0ACQCADDQAgCUECdCEKQQAhCwNAIAsgB0YNAyAGIAtBAnQiA2ogACADaiIMKgIAOAIAIAEhDSACIQ4gBSEPIAQhAyAJIQgCQANAIAhFDQEgAyADKgIAIA4qAgAgDCoCAJSSOAIAIA8gDCoCACANKgIAlDgCACANQQRqIQ0gDkEEaiEOIA9BBGohDyADQQRqIQMgCEF/aiEIDAALCyACIApqIQIgBSAKaiEFIAtBAWohCwwACwsgCEEANgIAIAhBBGohCCADQX9qIQMMAAsLC1MCAX8BfSOAgICAAEEQayIAJICAgIAAIABBgICAhwQ2AgxBBEEEQQRBoIDAgAAgAEEMahCCgICAACAAKgIMIQEgAEEQaiSAgICAACABQwAA4EBbC5ACAQJ/I4CAgIAAQdAAayIEJICAgIAAIARBIGpBCGpBACkCwIDAgAA3AwAgBEEwakEAKQLIgMCAADcDACAEQRBqQQhqQQAoArSAwIAANgIAIARCgIDAiISAgIjBADcDOCAEQQApAriAwIAANwMgIARBACkCrIDAgAA3AxAgBCABOAIMIAQgADgCCCAEQQE6AEwgBEEBOwFIIARCg4CAgCA3AkAgBEEIakHQgMCAAEGAgMCAACAEQcAAaiAEQRBqIARBIGogBEE4ahCEgICAACAEQRBqIQUCQAJAAkAgAg4CAgABCyAEQSBqIQUMAQsgBEE4aiEFCyAFIANBAnRqKgIAIQEgBEHQAGokgICAgAAgAQslACADIAAgASACQQAQgYCAgACUIAQgACABIAJBARCBgICAAJSSCwtlAQBBgIDAAAtcAACAPwAAAEAAAEBAAACAvwAAAAAAAIA/AAAAPwAAAL8AAAAAAAAAAAEAAAAAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAAIA/AAAAQAAAQEA=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function lossAt(x0, x1, x2, g0, g1) {
+ return W.t27_loss_at(x0, x1, x2, g0, g1);
+}
+function backwardAt(g0, g1, which, index) {
+ return W.t27_backward_at(g0, g1, int(which, 0, 255, "backward_at.which"), int(index, 0, 4294967295, "backward_at.index"));
+}
+function emptyForwardUntouched() {
+ return W.t27_empty_forward_untouched() !== 0;
+}
+function emptyBackwardUntouched() {
+ return W.t27_empty_backward_untouched() !== 0;
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+const __NOT_LOWERED__ = Object.freeze(["activate", "forward", "backward", "cfg", "forward_at"]);
+export {
+ __NOT_LOWERED__,
+ backwardAt,
+ emptyBackwardUntouched,
+ emptyForwardUntouched,
+ lossAt,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-backprop/logic.t27 b/apps/website/public/widgets/nn-backprop/logic.t27
new file mode 100644
index 0000000000..a75924f73d
--- /dev/null
+++ b/apps/website/public/widgets/nn-backprop/logic.t27
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Row-major weight layout for cache efficiency | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward and backward were empty placeholders, the tests asserted
+// `result != undefined`, and the invariants asserted `true` over an undeclared
+// `valid_input`. Dense is now y = act(W x + b) with W row-major
+// (output_size x input_size), and backward gives the three gradients.
+
+module Dense;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const MIN_INPUT_SIZE : u32 = 1;
+ const MAX_INPUT_SIZE : u32 = 4096;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const DenseConfig = struct {
+ input_size : u32,
+ output_size : u32,
+ has_bias : bool,
+ activation : ActivationType,
+ };
+
+ pub const ActivationType = enum(u8) {
+ None,
+ ReLU,
+ GELU,
+ Sigmoid,
+ Tanh,
+ Softmax,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // relu, the one activation the tests exercise besides None
+ fn activate(z: f32, activation: ActivationType) -> f32 {
+ if (activation == ActivationType.ReLU) {
+ if (z > 0.0) {
+ return z;
+ }
+ return 0.0;
+ }
+ return z;
+ }
+
+ // output[o] = act(sum_i weights[o * in + i] * input[i] + bias[o])
+ fn forward(input: []const f32, weights: []const f32, bias: []const f32, config: DenseConfig, output: []f32) -> void {
+ var o : usize = 0;
+ while (o < config.output_size) {
+ var z : f32 = 0.0;
+ if (config.has_bias) {
+ z = bias[o];
+ }
+ var i : usize = 0;
+ while (i < config.input_size) {
+ z = z + weights[o * config.input_size + i] * input[i];
+ i = i + 1;
+ }
+ output[o] = activate(z, config.activation);
+ o = o + 1;
+ }
+ }
+
+ // For a linear layer (activation None), given dL/dy = grad_output:
+ // grad_input[i] = sum_o W[o][i] grad_output[o] (W^T g)
+ // grad_weights[o][i] = grad_output[o] * input[i] (g x^T)
+ // grad_bias[o] = grad_output[o]
+ fn backward(grad_output: []const f32, input: []const f32, weights: []const f32, config: DenseConfig, grad_input: []f32, grad_weights: []f32, grad_bias: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.input_size) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var o : usize = 0;
+ while (o < config.output_size) {
+ grad_bias[o] = grad_output[o];
+ i = 0;
+ while (i < config.input_size) {
+ grad_input[i] = grad_input[i] + weights[o * config.input_size + i] * grad_output[o];
+ grad_weights[o * config.input_size + i] = grad_output[o] * input[i];
+ i = i + 1;
+ }
+ o = o + 1;
+ }
+ }
+
+ // The fixture: 3 inputs, 2 outputs, W = [[1, 2, 3], [-1, 0, 1]], b = [0.5, -0.5]
+ fn cfg(activation: ActivationType) -> DenseConfig {
+ return DenseConfig{ .input_size = 3, .output_size = 2, .has_bias = true, .activation = activation };
+ }
+
+ fn forward_at(x0: f32, x1: f32, x2: f32, activation: ActivationType, index: usize) -> f32 {
+ const input : [3]f32 = [x0, x1, x2];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ const bias : [2]f32 = [0.5, -0.5];
+ var output : [2]f32 = [0.0] ** 2;
+ forward(&input, &weights, &bias, cfg(activation), &output);
+ return output[index];
+ }
+
+ // L = g . y for g = (g0, g1): the loss whose gradients backward computes
+ fn loss_at(x0: f32, x1: f32, x2: f32, g0: f32, g1: f32) -> f32 {
+ return g0 * forward_at(x0, x1, x2, ActivationType.None, 0) + g1 * forward_at(x0, x1, x2, ActivationType.None, 1);
+ }
+
+ // which: 0 = grad_input, 1 = grad_weights, 2 = grad_bias
+ fn backward_at(g0: f32, g1: f32, which: u8, index: usize) -> f32 {
+ const grad_output : [2]f32 = [g0, g1];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ var grad_weights : [6]f32 = [9.0] ** 6;
+ var grad_bias : [2]f32 = [9.0] ** 2;
+ backward(&grad_output, &input, &weights, cfg(ActivationType.None), &grad_input, &grad_weights, &grad_bias);
+ if (which == 0) {
+ return grad_input[index];
+ }
+ if (which == 1) {
+ return grad_weights[index];
+ }
+ return grad_bias[index];
+ }
+
+ // An empty layer (no inputs, no outputs) writes nothing
+ fn empty_forward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ forward(&none, &none, &none, empty_cfg, &sentinel);
+ return sentinel[0] == 7.0;
+ }
+
+ fn empty_backward_untouched() -> bool {
+ const none : [0]f32 = [];
+ var sentinel : [1]f32 = [7.0];
+ const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
+ backward(&none, &none, &none, empty_cfg, &sentinel, &sentinel, &sentinel);
+ return sentinel[0] == 7.0;
+ }
+
+ fn valid_input() -> f32 {
+ return forward_at(1.0, 2.0, 3.0, ActivationType.None, 0);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // x = (1, 2, 3): y0 = 1 + 4 + 9 + 0.5 = 14.5, y1 = -1 + 0 + 3 - 0.5 = 1.5
+ test forward_basic_case
+ given y0 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 0)
+ when y1 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 1)
+ then y0 == 14.5 and y1 == 1.5
+
+ // ReLU clips the negative pre-activation: x = (0, 0, -1) gives y = (-2.5, -1.5) -> (0, 0)
+ test forward_relu_clips_negatives
+ given linear = forward_at(0.0, 0.0, -1.0, ActivationType.None, 0)
+ when clipped = forward_at(0.0, 0.0, -1.0, ActivationType.ReLU, 0)
+ then linear == -2.5 and clipped == 0.0 and forward_at(1.0, 2.0, 3.0, ActivationType.ReLU, 0) == 14.5
+
+ test forward_empty_input
+ given result = empty_forward_untouched()
+ then result
+
+ // g = (1, 2): grad_input = W^T g = (1 - 2, 2 + 0, 3 + 2) = (-1, 2, 5); grad_bias = g;
+ // grad_weights[1][2] = g1 * x2 = 6
+ test backward_basic_case
+ given gi0 = backward_at(1.0, 2.0, 0, 0)
+ when gi2 = backward_at(1.0, 2.0, 0, 2)
+ then gi0 == -1.0 and backward_at(1.0, 2.0, 0, 1) == 2.0 and gi2 == 5.0 and backward_at(1.0, 2.0, 2, 1) == 2.0 and backward_at(1.0, 2.0, 1, 5) == 6.0
+
+ // dL/dx1 by finite difference equals grad_input[1]
+ test backward_matches_finite_difference
+ given slope = (loss_at(1.0, 2.0 + 0.01, 3.0, 1.0, 2.0) - loss_at(1.0, 2.0 - 0.01, 3.0, 1.0, 2.0)) / 0.02
+ when result = backward_at(1.0, 2.0, 0, 1)
+ then @abs(result - slope) < 0.001
+
+ test backward_empty_gradient
+ given result = empty_backward_untouched()
+ then result
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.size() == config.input_size: the bounds the layer accepts
+ invariant dense_constraint_0
+ assert MIN_INPUT_SIZE >= 1 and MAX_INPUT_SIZE >= MIN_INPUT_SIZE
+
+ // output.size() == config.output_size: each output is one row of W x + b
+ invariant dense_constraint_1
+ assert valid_input() == 14.5
+
+ // weights.size() == input_size * output_size: the fixture holds 3 * 2 = 6 weights
+ invariant dense_constraint_2
+ assert cfg(ActivationType.None).input_size * cfg(ActivationType.None).output_size == 6
+
+ // bias.size() == output_size (if has_bias): the fixture carries a bias of 2
+ invariant dense_constraint_3
+ assert cfg(ActivationType.None).has_bias
+
+ // All inputs must be finite (no NaN/Inf): finite in, finite out
+ invariant dense_constraint_4
+ assert valid_input() == valid_input()
diff --git a/apps/website/public/widgets/nn-backprop/nn-backprop.t27 b/apps/website/public/widgets/nn-backprop/nn-backprop.t27
new file mode 100644
index 0000000000..9b1fde1101
--- /dev/null
+++ b/apps/website/public/widgets/nn-backprop/nn-backprop.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-backprop.t27 -- backpropagation through one dense layer
+; Source of truth for public/widgets/nn-backprop/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-backprop/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/dense_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 19 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets the output gradients g0 and g1 from -2 to 2. Edges are
+; backward_at(g0, g1, 1, o * 3 + i), input nodes backward_at(g0, g1, 0, i), bias tiles
+; backward_at(g0, g1, 2, o), and the loss loss_at = g . y at x and at x with x0 moved by 1.
+
+; WHAT IT DOES NOT CLAIM: the layer is the spec's fixture (W = [[1, 2, 3], [-1, 0, 1]], b = (0.5,
+; -0.5)), linear; the inputs are repeated as K_X.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_backprop;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-backprop";
+pub const TITLE : str = "Backpropagation: how blame flows back through a layer";
+pub const DESCRIPTION : str = "Set how much the loss wants each of two outputs to move: every weight gets that times its input, every input gets W transposed times it. The edges and nodes show the numbers.";
+pub const IMAGE_ALT : str = "Backprop card: a dense layer of 3 inputs and 2 outputs with g = (1, 0.5). Six edges are coloured by their weight gradients 1, 2, 3 and 0.5, 1, 1.5; the input nodes hold 0.5, 2.0 and 3.5. Tiles read input gradients 0.50 2.00 3.50, bias gradients 1.00 0.50, loss 15.250 and with x0 moved by 1 15.750.";
+pub const HOOK : str = "With g = (1, 0.5) on inputs (1, 2, 3), the weights get 1, 2, 3 and 0.5, 1, 1.5, and the inputs get 0.5, 2, 3.5. Two products are the whole rule.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/dense_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 19 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X : [3]u8 = [1, 2, 3];
+pub const K_X_NUDGED : [3]u8 = [2, 2, 3];
+pub const K_G_LO_C : i16 = -200;
+pub const K_G_HI_C : u16 = 200;
+pub const K_PRESET_G0_C : [4]i16 = [100, 100, 0, -200];
+pub const K_PRESET_G1_C : [4]i16 = [50, 0, 100, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["g = (1, 0.5)", "Only output 0", "Only output 1", "g = (-2, 1)"];
+pub const SAY_ASK : str = "How does blame for an error flow back through a layer?";
+pub const SAY_G : str = "g";
+pub const SAY_NET_LABEL : str = "Three inputs, two outputs; edges coloured by their weight gradient, inputs by theirs";
+pub const SAY_NET_CAP : str = "Edge label: g_o x_i. Input node: sum over o of W[o][i] g_o. Pink is negative, green positive.";
+pub const SAY_GRAD_IN : str = "input gradients";
+pub const SAY_GRAD_B : str = "bias gradients";
+pub const SAY_LOSS : str = "loss g . y";
+pub const SAY_LOSS_NUDGED : str = "with x0 + 1";
+pub const SAY_NOTE : str = "The layer is linear, so moving x0 by 1 moves the loss by exactly its input gradient: compare the last two tiles. Chain this rule layer by layer and you have backpropagation.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/dense_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_inputs_are_the_spec_fixture {
+ assert K_X[0] == 1;
+ assert K_X[2] == 3;
+ assert K_X_NUDGED[0] == K_X[0] + 1;
+ assert K_X_NUDGED[2] == K_X[2];
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_G0_C[3] == K_G_LO_C;
+ assert K_PRESET_G1_C[2] <= K_G_HI_C;
+ assert K_G_LO_C + K_G_HI_C == 0;
+}
diff --git a/apps/website/public/widgets/nn-backprop/tool.js b/apps/website/public/widgets/nn-backprop/tool.js
new file mode 100644
index 0000000000..8f5522d17c
--- /dev/null
+++ b/apps/website/public/widgets/nn-backprop/tool.js
@@ -0,0 +1,39 @@
+// nn-backprop/tool.js -- lesson 19 of neural-networks: how blame flows back through a layer. Given
+// how much the loss wants each output to change (g), backpropagation hands each weight g times its
+// input and each input W^T g. Every edge and node value is backward_at, the loss loss_at of
+// gHashTag/t27 specs/ml/layers/dense_layer.t27, compiled to wasm (./logic.js): 3 inputs (1, 2, 3)
+// (K_X, the spec's fixture), 2 outputs. Words are SAY_ and numbers K_ constants of
+// specs/widgets/nn-backprop.t27.
+import { backwardAt, lossAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, el, fmt, net, root, say, source, tiles, tone } from '../nn-kit.js'
+
+const X = K('K_X', []), XN = K('K_X_NUDGED', [])
+const G0 = centi(K('K_G_LO_C', -200)), G1 = centi(K('K_G_HI_C', 200))
+const ctl = controls([
+ { key: 'g0', label: `${say('SAY_G')} 0`, min: G0, max: G1, step: 0.05, fmt: (v) => fmt(v, 2) },
+ { key: 'g1', label: `${say('SAY_G')} 1`, min: G0, max: G1, step: 0.05, fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { g0: K('K_PRESET_G0_C', []).map(centi), g1: K('K_PRESET_G1_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-net', say('SAY_NET_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, svg, el('p', { class: 'nn-cap' }, say('SAY_NET_CAP')), stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const gi = [0, 1, 2].map((i) => backwardAt(s.g0, s.g1, 0, i))
+ const inputs = X.map((x, i) => ({ label: `x${i} = ${x}`, value: fmt(gi[i], 2), color: tone(gi[i], 6) }))
+ const outputs = [s.g0, s.g1].map((g, o) => ({ label: `${say('SAY_G')} ${o}`, value: fmt(g, 2), color: tone(g, 2) }))
+ const edges = []
+ for (let o = 0; o < 2; o++) for (let i = 0; i < 3; i++) {
+ const v = backwardAt(s.g0, s.g1, 1, o * 3 + i)
+ edges.push({ from: [0, i], to: [1, o], value: v, label: fmt(v, 2) })
+ }
+ net(svg, [inputs, outputs], edges, { w: 440, h: 360, m: 6 })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_GRAD_IN'), value: gi.map((v) => fmt(v, 2)).join(' '), tone: 0 },
+ { label: say('SAY_GRAD_B'), value: [0, 1].map((o) => fmt(backwardAt(s.g0, s.g1, 2, o), 2)).join(' '), tone: 1 },
+ { label: say('SAY_LOSS'), value: fmt(lossAt(X[0], X[1], X[2], s.g0, s.g1), 3), tone: 2 },
+ { label: say('SAY_LOSS_NUDGED'), value: fmt(lossAt(XN[0], XN[1], XN[2], s.g0, s.g1), 3), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-batchnorm/card.png b/apps/website/public/widgets/nn-batchnorm/card.png
new file mode 100644
index 0000000000..9bb331e1c7
Binary files /dev/null and b/apps/website/public/widgets/nn-batchnorm/card.png differ
diff --git a/apps/website/public/widgets/nn-batchnorm/index.html b/apps/website/public/widgets/nn-batchnorm/index.html
new file mode 100644
index 0000000000..6a4920cd43
--- /dev/null
+++ b/apps/website/public/widgets/nn-batchnorm/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Batch norm: features on very different scales, made comparable · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Batch norm: features on very different scales, made comparable
+
A batch of 4 samples whose 3 features live on scales 1, 10 and 1: batch norm gives each feature mean 0 and variance 1. Then set gamma and the running statistics for inference.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-batchnorm/logic.js b/apps/website/public/widgets/nn-batchnorm/logic.js
new file mode 100644
index 0000000000..30a29340cc
--- /dev/null
+++ b/apps/website/public/widgets/nn-batchnorm/logic.js
@@ -0,0 +1,36 @@
+// Generated by scripts/t27-logic.mjs from batchnorm_layer.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 479b0fe4e0c7103ae57150096356ef13b6bdc796f6189d8cb563e4d26c115254
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-batchnorm/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABEgNgAAF/YAJ/fwF9YAN9f38BfQMFBAABAQIEBQFwAQEBBQMBABEGCQF/AUGAgMAACwdOBQZtZW1vcnkCAA90MjdfdmFsaWRfaW5wdXQAAAx0MjdfdHJhaW5fYXQAAQ90MjdfYmFja3dhcmRfYXQAAg50MjdfZm9yd2FyZF9hdAADCowGBAQAQQMLtQIEBH8BfQF/An0jgICAgABBMGsiAiSAgICAAEEAIQMCQEEwRQ0AIAJBAEEw/AsAC0GAgMCAACEEIAIhBQJAA0AgA0EDRg0BQwAAAAAhBkEAIQcCQANAIAdBMEYNASAGIAQgB2oqAgCSIQYgB0EMaiEHDAALCyAGQwAAgD6UIQhDAAAAACEGQQAhBwJAA0AgB0EwRg0BIAYgBCAHaioCACAIkyIJIAmUkiEGIAdBDGohBwwACwsgBkMAAIA+lEOsxSc3kpEhBkEAIQcCQANAIAdBMEYNASAFIAdqIAQgB2oqAgAgCJMgBpVDAAAAAJI4AgAgB0EMaiEHDAALCyAFQQRqIQUgBEEEaiEEIANBAWohAwwACwsgAiAAQQxsaiABQQJ0aioCACEGIAJBMGokgICAgAAgBgumAQICfwF9I4CAgIAAQTBrIgIkgICAgAAgAkGs////AzYCJCACQqz////D9f//PzcCKCACQYCAgPwDNgIYIAJCgICA/IOAgMA/NwIcIAJCrP//+8P1//8/NwIMIAJBwf//gQQ2AhQgAkEMaiEDAkACQAJAIAAOAgIAAQsgAkEYaiEDDAELIAJBJGohAwsgAyABQQJ0aioCACEEIAJBMGokgICAgAAgBAulAgECfyOAgICAAEHAAGsiAySAgICAACADQYCAgPwDNgIsIANCgICA/IOAgMA/NwIkQQAhBCADQQA2AiAgA0IANwIYIAMgADgCFCADIAA4AhAgAyAAOAIMAkAgAUEBcUUNACADQYCAgIAENgIgIANCgICAgISAgIDAADcCGCADQYCAgIQENgIsIANCgICAhISAgMDAADcCJAsgA0E4akEANgIAIANCADcDMAJAA0AgBEEMRg0BIANBMGogBGogA0EMaiAEaioCACAEQbCAwIAAaioCACADQRhqIARqKgIAkyADQSRqIARqKgIAQ6zFJzeSkZWUQwAAgD6SOAIAIARBBGohBAwACwsgA0EwaiACQQJ0aioCACEAIANBwABqJICAgIAAIAALC0UBAEGAgMAACzwAAIA/AAAgQQAAgL8AAABAAACgQQAAgL8AAEBAAADwQQAAgD8AAIBAAAAgQgAAgD8AAIA/AAAAQAAAQEA=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function forwardAt(g, shifted, index) {
+ return W.t27_forward_at(g, shifted ? 1 : 0, int(index, 0, 4294967295, "forward_at.index"));
+}
+function backwardAt(which, index) {
+ return W.t27_backward_at(int(which, 0, 255, "backward_at.which"), int(index, 0, 4294967295, "backward_at.index"));
+}
+function trainAt(b, f) {
+ return W.t27_train_at(int(b, 0, 4294967295, "train_at.b"), int(f, 0, 4294967295, "train_at.f"));
+}
+function validInput() {
+ return W.t27_valid_input() >>> 0;
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward", "forward_train", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ backwardAt,
+ forwardAt,
+ trainAt,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-batchnorm/logic.t27 b/apps/website/public/widgets/nn-batchnorm/logic.t27
new file mode 100644
index 0000000000..e910bfe464
--- /dev/null
+++ b/apps/website/public/widgets/nn-batchnorm/logic.t27
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Prevent division by zero in sqrt | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward copied one element and backward zeroed two gradients
+// (placeholders), and the tests pinned those placeholders: output[0] == input[0],
+// grad_gamma[0] == grad_beta[0] == 0. They now compute batch normalisation:
+// forward/backward in inference form, normalising by the running statistics, and
+// forward_train normalising by the batch's own mean and variance.
+
+module Batchnorm;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_EPSILON : f32 = 1.0e-5;
+ const DEFAULT_MOMENTUM : f32 = 0.1;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const BatchNormConfig = struct {
+ num_features : u32,
+ epsilon : f32,
+ momentum : f32,
+ affine : bool,
+ track_running_stats : bool,
+ };
+
+ pub const BatchNormState = struct {
+ running_mean : []f32,
+ running_var : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Inference: output[f] = gamma[f] (input[f] - running_mean[f]) / sqrt(running_var[f] + eps) + beta[f]
+ fn forward(input: []const f32, gamma: []const f32, beta: []const f32, state: BatchNormState, output: []f32, config: BatchNormConfig) -> void {
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const x_hat = (input[f] - state.running_mean[f]) / sqrt(state.running_var[f] + config.epsilon);
+ output[f] = gamma[f] * x_hat + beta[f];
+ f = f + 1;
+ }
+ }
+
+ // Inference backward (mean and variance are constants here):
+ // grad_input[f] = grad_output[f] gamma[f] / sqrt(var[f] + eps)
+ // grad_gamma[f] = grad_output[f] x_hat[f], grad_beta[f] = grad_output[f]
+ fn backward(grad_output: []const f32, input: []const f32, gamma: []const f32, beta: []const f32, mean: []const f32, var: []const f32, grad_gamma: []f32, grad_beta: []f32, grad_input: []f32, config: BatchNormConfig) -> void {
+ var f : usize = 0;
+ while (f < config.num_features) {
+ const inv_std = 1.0 / sqrt(var[f] + config.epsilon);
+ grad_gamma[f] = grad_output[f] * (input[f] - mean[f]) * inv_std;
+ grad_beta[f] = grad_output[f];
+ grad_input[f] = grad_output[f] * gamma[f] * inv_std;
+ f = f + 1;
+ }
+ }
+
+ // Training: input is batch x num_features, row-major. Each feature is normalised by
+ // its own batch mean and (biased) batch variance, then scaled and shifted.
+ fn forward_train(input: []const f32, batch: usize, gamma: []const f32, beta: []const f32, output: []f32, config: BatchNormConfig) -> void {
+ const nf : usize = config.num_features;
+ var f : usize = 0;
+ while (f < nf) {
+ var mean : f32 = 0.0;
+ var b : usize = 0;
+ while (b < batch) {
+ mean = mean + input[b * nf + f];
+ b = b + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(batch));
+ var variance : f32 = 0.0;
+ b = 0;
+ while (b < batch) {
+ const d = input[b * nf + f] - mean;
+ variance = variance + d * d;
+ b = b + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(batch));
+ b = 0;
+ while (b < batch) {
+ output[b * nf + f] = gamma[f] * (input[b * nf + f] - mean) / sqrt(variance + config.epsilon) + beta[f];
+ b = b + 1;
+ }
+ f = f + 1;
+ }
+ }
+
+ fn cfg() -> BatchNormConfig {
+ return BatchNormConfig{ .num_features = 3, .epsilon = DEFAULT_EPSILON, .momentum = DEFAULT_MOMENTUM, .affine = true, .track_running_stats = true };
+ }
+
+ // forward on input (1, 2, 3), gamma g, beta 0.25, running mean 0 / var 1 (or mean 2 / var 4)
+ fn forward_at(g: f32, shifted: bool, index: usize) -> f32 {
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [g, g, g];
+ const beta : [3]f32 = [0.25, 0.25, 0.25];
+ var mean : [3]f32 = [0.0, 0.0, 0.0];
+ var variance : [3]f32 = [1.0, 1.0, 1.0];
+ if (shifted) {
+ mean = [2.0, 2.0, 2.0];
+ variance = [4.0, 4.0, 4.0];
+ }
+ const state = BatchNormState{ .running_mean = &mean, .running_var = &variance };
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&input, &gamma, &beta, state, &output, cfg());
+ return output[index];
+ }
+
+ // which: 0 = grad_gamma, 1 = grad_beta, 2 = grad_input; grad_output 1, gamma 2, mean 0, var 1
+ fn backward_at(which: u8, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [1.0, 2.0, 3.0];
+ const gamma : [3]f32 = [2.0, 2.0, 2.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ const mean : [3]f32 = [0.0, 0.0, 0.0];
+ const variance : [3]f32 = [1.0, 1.0, 1.0];
+ var grad_gamma : [3]f32 = [9.0] ** 3;
+ var grad_beta : [3]f32 = [9.0] ** 3;
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &gamma, &beta, &mean, &variance, &grad_gamma, &grad_beta, &grad_input, cfg());
+ if (which == 0) {
+ return grad_gamma[index];
+ }
+ if (which == 1) {
+ return grad_beta[index];
+ }
+ return grad_input[index];
+ }
+
+ // forward_train on a batch of 4 samples x 3 features; returns output[b][f]
+ fn train_at(b: usize, f: usize) -> f32 {
+ const input : [12]f32 = [1.0, 10.0, -1.0, 2.0, 20.0, -1.0, 3.0, 30.0, 1.0, 4.0, 40.0, 1.0];
+ const gamma : [3]f32 = [1.0, 1.0, 1.0];
+ const beta : [3]f32 = [0.0, 0.0, 0.0];
+ var output : [12]f32 = [0.0] ** 12;
+ forward_train(&input, 4, &gamma, &beta, &output, cfg());
+ return output[b * 3 + f];
+ }
+
+ fn valid_input() -> u32 {
+ return cfg().num_features;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // With mean 0, var 1, gamma 1, beta 0 the layer is the identity up to eps:
+ // 1 / sqrt(1 + 1e-5) = 0.999995 (the old `==` held only for the copy placeholder)
+ test "forward_basic_case"
+ given out0 = forward_at(1.0, false, 0) - 0.25
+ when out2 = forward_at(1.0, false, 2) - 0.25
+ then @abs(out0 - 1.0) < 0.00001 and @abs(out2 - 3.0) < 0.00003
+
+ // mean 2, var 4: (x - 2) / 2 * gamma + beta, so x = 3 with gamma 2 gives 1.25
+ test "forward_uses_running_statistics"
+ given result = forward_at(2.0, true, 2)
+ when centre = forward_at(2.0, true, 1)
+ then @abs(result - 1.25) < 0.00001 and centre == 0.25
+
+ // grad_gamma = g x_hat = (1, 2, 3) / sqrt(1 + eps), grad_beta = g = 1,
+ // grad_input = g gamma / sqrt(1 + eps) = 2 (the old 0, 0 were the placeholder's)
+ test "backward_basic_case"
+ given grad_gamma = backward_at(0, 2)
+ when grad_input = backward_at(2, 0)
+ then @abs(grad_gamma - 3.0) < 0.0001 and backward_at(1, 0) == 1.0 and grad_input != 0.0 and @abs(grad_input - 2.0) < 0.0001
+
+ // Each feature of the batch comes out with mean 0 and variance 1
+ test "forward_train_normalises_each_feature"
+ given mean_f1 = train_at(0, 1) + train_at(1, 1) + train_at(2, 1) + train_at(3, 1)
+ when var_f1 = (train_at(0, 1) * train_at(0, 1) + train_at(1, 1) * train_at(1, 1) + train_at(2, 1) * train_at(2, 1) + train_at(3, 1) * train_at(3, 1)) / 4.0
+ then @abs(mean_f1) < 0.00001 and @abs(var_f1 - 1.0) < 0.0001 and @abs(train_at(0, 0) - train_at(0, 1)) < 0.00001 and @abs(train_at(0, 2) + 1.0) < 0.0001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // num_features > 0
+ invariant batchnorm_constraint_0
+ assert valid_input() > 0
+
+ // epsilon > 0
+ invariant batchnorm_constraint_1
+ assert DEFAULT_EPSILON > 0.0
+
+ // 0 <= momentum < 1
+ invariant batchnorm_constraint_2
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+
+ // gamma.size() == num_features: the fixtures carry 3 of each
+ invariant batchnorm_constraint_3
+ assert valid_input() == 3
+
+ // beta.size() == num_features: beta shifts every feature, so x = mean gives exactly beta
+ invariant batchnorm_constraint_4
+ assert forward_at(2.0, true, 1) == 0.25
diff --git a/apps/website/public/widgets/nn-batchnorm/nn-batchnorm.t27 b/apps/website/public/widgets/nn-batchnorm/nn-batchnorm.t27
new file mode 100644
index 0000000000..5c886e03c8
--- /dev/null
+++ b/apps/website/public/widgets/nn-batchnorm/nn-batchnorm.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-batchnorm.t27 -- batch normalisation: features on different scales, made comparable
+; Source of truth for public/widgets/nn-batchnorm/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-batchnorm/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/batchnorm_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 12 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader reads the batch before and after and sets gamma and the running
+; statistics. The normalised batch is train_at(b, f) and the inference bars forward_at(gamma,
+; shifted, i) on inputs (1, 2, 3), beta 0.25.
+
+; WHAT IT DOES NOT CLAIM: the batch on the left is the spec's fixture, repeated as K_BATCH; the
+; normalised values come from the compiled fn.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_batchnorm;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-batchnorm";
+pub const TITLE : str = "Batch norm: features on very different scales, made comparable";
+pub const DESCRIPTION : str = "A batch of 4 samples whose 3 features live on scales 1, 10 and 1: batch norm gives each feature mean 0 and variance 1. Then set gamma and the running statistics for inference.";
+pub const IMAGE_ALT : str = "Batch-norm card: on the left the spec's batch, 4 samples by 3 features (1, 2, 3, 4 / 10, 20, 30, 40 / -1, -1, 1, 1); on the right the normalised batch, the first two columns both -1.34, -0.45, 0.45, 1.34 and the third -1.00, -1.00, 1.00, 1.00. Below, gamma 1 at inference gives 1.25, 2.25, 3.25.";
+pub const HOOK : str = "Features 1..4 and 10..40 come out identical, -1.34, -0.45, 0.45, 1.34: batch norm keeps the shape of each feature and throws away its scale.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/batchnorm_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 12 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_BATCH_ROWS : u8 = 4;
+pub const K_FEATURES : u8 = 3;
+pub const K_BATCH : [12]i8 = [1, 10, -1, 2, 20, -1, 3, 30, 1, 4, 40, 1];
+pub const K_GAMMA_HI_C : u16 = 300;
+pub const K_INFER_INPUT : [3]u8 = [1, 2, 3];
+pub const K_BAR_LO_C : i16 = -200;
+pub const K_BAR_HI_C : u16 = 1000;
+pub const K_BAR_TICKS : [5]i8 = [-2, 0, 2, 5, 10];
+pub const K_PRESET_GAMMA_C : [4]u16 = [100, 200, 100, 50];
+pub const K_PRESET_SHIFTED : [4]u8 = [0, 0, 1, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["gamma 1, mean 0 / var 1", "gamma 2", "gamma 1, mean 2 / var 4", "gamma 0.5, mean 2 / var 4"];
+pub const SAY_ASK : str = "What happens when one feature is ten times larger than another?";
+pub const SAY_GAMMA : str = "gamma";
+pub const SAY_SHIFTED : str = "running mean 2, variance 4";
+pub const SAY_BEFORE_LABEL : str = "The batch: 4 samples x 3 features";
+pub const SAY_AFTER_LABEL : str = "After batch norm";
+pub const SAY_INFER_LABEL : str = "At inference: inputs 1, 2, 3 through the running statistics";
+pub const SAY_FEATURE : str = "feature";
+pub const SAY_SAMPLE : str = "sample";
+pub const SAY_INPUT : str = "input";
+pub const SAY_COL : str = "feature {0} after";
+pub const SAY_INFER : str = "inference out";
+pub const SAY_NOTE : str = "Training normalises with the batch's own mean and variance; inference cannot, so it uses running averages kept during training. gamma and beta are learned, so the layer can undo the normalisation where that helps.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/batchnorm_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_batch_is_the_spec_fixture {
+ assert K_BATCH_ROWS * K_FEATURES == 12;
+ assert K_BATCH[1] == 10;
+ assert K_BATCH[10] == 40;
+ assert K_BATCH[2] + 1 == 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_GAMMA_C[1] <= K_GAMMA_HI_C;
+ assert K_PRESET_SHIFTED[2] == 1;
+ assert K_INFER_INPUT[2] == 3;
+}
diff --git a/apps/website/public/widgets/nn-batchnorm/tool.js b/apps/website/public/widgets/nn-batchnorm/tool.js
new file mode 100644
index 0000000000..db17b14c82
--- /dev/null
+++ b/apps/website/public/widgets/nn-batchnorm/tool.js
@@ -0,0 +1,40 @@
+// nn-batchnorm/tool.js -- lesson 12 of neural-networks: how a network copes with features on very
+// different scales. Batch normalisation subtracts each feature's batch mean and divides by its batch
+// spread, then scales by gamma and shifts by beta. Every normalised value is train_at, every
+// inference value forward_at of gHashTag/t27 specs/ml/layers/batchnorm_layer.t27, compiled to wasm
+// (./logic.js). The batch on the left is the spec's own (K_BATCH, held equal to it by the widget's
+// tests). Words are SAY_ and numbers K_ constants of specs/widgets/nn-batchnorm.t27.
+import { forwardAt, trainAt } from './logic.js'
+import { COLORS, K, W, ask, bars, canvas, centi, controls, el, fill, fmt, heat, root, say, source, tiles, tone } from '../nn-kit.js'
+
+const B = K('K_BATCH_ROWS', 4), F = K('K_FEATURES', 3)
+const BATCH = K('K_BATCH', [])
+const ctl = controls([
+ { key: 'g', label: say('SAY_GAMMA'), min: 0, max: centi(K('K_GAMMA_HI_C', 300)), step: 0.05, fmt: (v) => fmt(v, 2) },
+ { key: 'shifted', label: say('SAY_SHIFTED'), kind: 'toggle' },
+], { names: W.SAY_PRESET_NAMES, values: { g: K('K_PRESET_GAMMA_C', []).map(centi), shifted: K('K_PRESET_SHIFTED', []) } }, draw)
+
+const before = canvas('nn-heat', say('SAY_BEFORE_LABEL'))
+const after = canvas('nn-heat', say('SAY_AFTER_LABEL'))
+const inf = canvas('nn-bars', say('SAY_INFER_LABEL'))
+const stat = el('div')
+const cols = [...Array(F)].map((_, f) => `${say('SAY_FEATURE')} ${f}`)
+const rows = [...Array(B)].map((_, b) => `${say('SAY_SAMPLE')} ${b}`)
+root.replaceChildren(ask(say('SAY_ASK')),
+ el('div', { class: 'nn-two' }, el('div', {}, el('p', { class: 'dk-label' }, say('SAY_BEFORE_LABEL')), before), el('div', {}, el('p', { class: 'dk-label' }, say('SAY_AFTER_LABEL')), after)),
+ stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), ctl.node, el('p', { class: 'dk-label' }, say('SAY_INFER_LABEL')), inf, source())
+
+function draw(s) {
+ heat(before, { rows: B, cols: F, value: (r, c) => BATCH[r * F + c], color: (v) => tone(v, 40), rowLabels: rows, colLabels: cols, w: 400, cell: 96, fmt: (v) => fmt(v, 0) })
+ heat(after, { rows: B, cols: F, value: (r, c) => trainAt(r, c), color: (v) => tone(v, 1.5), rowLabels: rows, colLabels: cols, w: 400, cell: 96, fmt: (v) => fmt(v, 2) })
+ const y = [0, 1, 2].map((i) => forwardAt(s.g, !!s.shifted, i))
+ bars(inf, { values: y, labels: K('K_INFER_INPUT', []).map((x) => `${say('SAY_INPUT')} ${x}`), colors: [COLORS[2], COLORS[0], COLORS[1]], lo: centi(K('K_BAR_LO_C', -200)), hi: centi(K('K_BAR_HI_C', 1000)), h: 240, ticks: K('K_BAR_TICKS', []), fmt: (v) => fmt(v, 3) })
+ stat.replaceChildren(tiles([
+ { label: fill(say('SAY_COL'), 0), value: [0, 1, 2, 3].map((b) => fmt(trainAt(b, 0), 2)).join(' '), tone: 0 },
+ { label: fill(say('SAY_COL'), 1), value: [0, 1, 2, 3].map((b) => fmt(trainAt(b, 1), 2)).join(' '), tone: 1 },
+ { label: fill(say('SAY_COL'), 2), value: [0, 1, 2, 3].map((b) => fmt(trainAt(b, 2), 2)).join(' '), tone: 2 },
+ { label: say('SAY_INFER'), value: y.map((v) => fmt(v, 2)).join(' '), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-bce/card.png b/apps/website/public/widgets/nn-bce/card.png
new file mode 100644
index 0000000000..5b1853131e
Binary files /dev/null and b/apps/website/public/widgets/nn-bce/card.png differ
diff --git a/apps/website/public/widgets/nn-bce/index.html b/apps/website/public/widgets/nn-bce/index.html
new file mode 100644
index 0000000000..39a6037ab6
--- /dev/null
+++ b/apps/website/public/widgets/nn-bce/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Cross-entropy: why a confident wrong answer costs so much · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Cross-entropy: why a confident wrong answer costs so much
+
Set three predicted probabilities against the answers 1, 0, 1: the loss is -log of the chance given to the truth. Drag toward a confident mistake and the cost climbs without bound.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-bce/logic.js b/apps/website/public/widgets/nn-bce/logic.js
new file mode 100644
index 0000000000..c5bfba2dd1
--- /dev/null
+++ b/apps/website/public/widgets/nn-bce/logic.js
@@ -0,0 +1,44 @@
+// Generated by scripts/t27-logic.mjs from binary_crossentropy_loss.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 4b041a9197282c33abafdcc293c3864bae585c9e417d743e0081cf4d734fcf9b
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-bce/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABLgdgAAF9YAR9fX1/AX1gA319fwF9YAV9fX1/fwF9YAJ9fQF9YAF9AX1gAn1/AX0DDAsAAQIAAwEEBQUFBgQFAXABAQEFAwEAEQYJAX8BQYCAwAALB2IHBm1lbW9yeQIAD3QyN192YWxpZF9pbnB1dAAADnQyN19iYXRjaF9sb3NzAAMLdDI3X2dyYWRfb2YABAt0MjdfbG9zc19vZgAFCHQyN19jbGlwAAYLdDI3X3NpZ21vaWQABwrFDgsZAEPNzEw/Q5qZmT5DZmZmP0EAEIGAgIAAC6oBAQJ/I4CAgIAAQSBrIgQkgICAgAAgBCACOAIQIAQgATgCDCAEIAA4AgggBEGV/9qeAzYCFEEAIQUgBEEAOgAZIAQgA0EBcSIDOgAYIAQgAzoAHEMAAAAAIQICQANAIAVBDEYNASACIARBCGogBWoqAgAgBUGAgMCAAGoqAgAgBEEUahCCgICAAJIhAiAFQQRqIQUMAAsLIARBIGokgICAgAAgAkMAAEBAlQunAQECfSACKgIAIQMCQCACLQAEQQFxRQ0AAkACQCAAQwAAAABgRQ0AQwAAgD8hBCAAjBCJgICAACEADAELIAAQiYCAgAAiACEECyAEIABDAACAP5KVIQALAkACQCAAIANdRQ0AIAMhAAwBCyAAQwAAgD8gA5MiA15FDQAgAyEAC0MAAIA/IACTEIiAgIAAIQMgASAAEIiAgIAAlCADQwAAgD8gAZOUkowLhQEDAX8BfQF/I4CAgIAAQRBrIgAkgICAgAAgAEGV/9qeAzYCBCAAQQA7AQhDAAAAACEBQXAhAgJAA0AgAkUNASABIAJBnIDAgABqKgIAIAJBrIDAgABqKgIAIABBBGoQgoCAgACSIQEgAkEEaiECDAALCyAAQRBqJICAgIAAIAFDAACAPpQLzgIBAn8jgICAgABBIGsiBSSAgICAAEEAIQYgBUEYakEAKAK0gMCAADYCACAFIAI4AgwgBSABOAIIIAUgADgCBCAFQQApAqyAwIAANwMQIANBAXEhAwJAA0AgBkEMRg0BIAZBgIDAgABqKgIAIQICQAJAIANFDQACQAJAIAVBBGogBmoqAgAiAUMAAAAAYEUNAEMAAIA/IQAgAYwQiYCAgAAhAQwBCyABEImAgIAAIgEhAAsgBUEQaiAGaiAAIAFDAACAP5KVIAKTOAIADAELQ5W/1jMhAQJAIAVBBGogBmoqAgAiAEOVv9YzXQ0AIAAhASAAQ/7/fz9eRQ0AQ/7/fz8hAQsgBUEQaiAGakMAAIA/IAKTQwAAgD8gAZOVIAIgAZWTOAIACyAGQQRqIQYMAAsLIAVBEGogBEECdGoqAgAhAiAFQSBqJICAgIAAIAILEAAgACABIAIgAxCBgICAAAspAAJAIAAgAV1FDQAgAQ8LAkAgAEMAAIA/IAGTIgFeRQ0AIAEhAAsgAAtAAQF9AkACQCAAQwAAAABgRQ0AQwAAgD8hASAAjBCJgICAACEADAELIAAQiYCAgAAiACEBCyABIABDAACAP5KVC4ICAgJ/An0CQAJAAkAgALwiAUGAgIAETg0AQwAAgP9DAADAfyAAQwAAAABbIgIbIQMgAg0CIAFBAEgNAiAAQwAAAEyUvCEBQeh+IQIMAQsgACEDIAFB////+wdLDQFBgX8hAkMAAAAAIQMgAUGAgID8A0YNAQsgAiABQY32qwJqIgFBF3ZqsiIDQ4BxMT+UIAFB////A3FB84nU+QNqvkMAAIC/kiIAIAND0fcXN5QgACAAQwAAAECSlSIDIAAgAEMAAAA/lJQiBCADIAOUIgAgACAAlCIAQ+7pkT6UQ6qqKj+SlCAAIABDJp54PpRDE87MPpKUkpKUkiAEk5KSIQMLIAML1AICA38DfQJAAkAgACAAXA0AIAC8IgFBH3YhAgJAAkACQAJAAkACQAJAIAFB/////wdxIgNBz9i6lQRLDQAgA0GY5MX1A00NASADQZKrlPwDTQ0CDAQLIANBgICA/AdLDQYCQCABQQBIDQAgA0GY5MWVBEkNACAAQwAAAH+UDwsgAUF/Sg0DQwAAAAAhBCADQbTjv5YESw0HDAMLIANBgICAyANNDQFBACEDQwAAAAAhBSAAIQQMBAsgAkEBcyACayEDDAILIABDAACAP5IPCyAAQzuquD+UIAJBAnQqAriAwIAAkvwAIQMLIAAgA7IiBEMAcjG/lJIiACAEQ46+vzWUIgWTIQQLIAAgBCAEIAQgBJQiBiAGQxVSNbuUQ4+qKj6SlJMiBpRDAAAAQCAGk5UgBZOSQwAAgD+SIQQgA0UNASAEIAMQioCAgAAPCyAAIQQLIAQLpgIBBH8CQAJAIAAgAFwNACAAvCICQf////8HcUGAgID8B08NACACQYCAgIB4cSEDQQggAkEBdCIEZ2sgBEEYdiAEQYCAgAhJGyEEAkAgAUEASA0AAkAgAUH+ASAEa0wNAEMAAIB/IACYDwsCQCAEIAFqIgVBAEoNACACIAF0IANyvg8LAkAgBEEASg0AIAJBASAEa3RB////A3EgBUEXdHIgA3K+DwsgAUEXdCACar4PCyABQQAgBGtKDQECQCABQWkgBGtODQAgA74PC0GAgIAEQQAgBEEAShsgAkH///8DcXJBACABayIDQQEgASAEamsiASADIAFIG0F/aiIBdkEBakEBdiACaCABQR9xRkF/c3G+IACYIQALIAAPCyABQRd0IAJqvgsLSQEAQYCAwAALQAAAgD8AAAAAAACAP83MTD+amZk+ZmZmP83MzD0AAIA/AAAAAAAAgD8AAAAAAAAQQQAAEEEAABBBAAAAPwAAAL8=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function sigmoid(x) {
+ return W.t27_sigmoid(x);
+}
+function clip(p, eps) {
+ return W.t27_clip(p, eps);
+}
+function lossOf(a, b, c, fromLogits) {
+ return W.t27_loss_of(a, b, c, fromLogits ? 1 : 0);
+}
+function gradOf(a, b, c, fromLogits, index) {
+ return W.t27_grad_of(a, b, c, fromLogits ? 1 : 0, int(index, 0, 4294967295, "grad_of.index"));
+}
+function batchLoss() {
+ return W.t27_batch_loss();
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+const __NOT_LOWERED__ = Object.freeze(["element_loss", "forward", "forward_batch", "gradient", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ batchLoss,
+ clip,
+ gradOf,
+ lossOf,
+ sigmoid,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-bce/logic.t27 b/apps/website/public/widgets/nn-bce/logic.t27
new file mode 100644
index 0000000000..063ee794ce
--- /dev/null
+++ b/apps/website/public/widgets/nn-bce/logic.t27
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Gradient w.r.t predictions | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: never compiled (`new f32[n]`, a float loop index, `predictions.size`,
+// an empty enum), and it approximated sigmoid and log with polynomials that
+// were 0.3 off at x = 1. The probability gradient also had the wrong sign on its
+// second term: d/dp of -(t log p + (1-t) log(1-p)) is -t/p + (1-t)/(1-p). It now
+// uses exp and log, writes gradients into a caller's buffer, and reduces by mean.
+
+module BinaryCe;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_EPSILON : f32 = 1e-7;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const BinaryCEConfig = struct {
+ from_logits : bool,
+ epsilon : f32,
+ reduction : Reduction,
+ };
+
+ pub const Reduction = enum(u8) {
+ Mean,
+ Sum,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn sigmoid(x: f32) -> f32 {
+ if (x >= 0.0) {
+ return 1.0 / (1.0 + exp(-x));
+ }
+ const e = exp(x);
+ return e / (1.0 + e);
+ }
+
+ // p clipped into [eps, 1 - eps] so log stays finite
+ fn clip(p: f32, eps: f32) -> f32 {
+ if (p < eps) {
+ return eps;
+ }
+ if (p > 1.0 - eps) {
+ return 1.0 - eps;
+ }
+ return p;
+ }
+
+ // -(t log p + (1 - t) log(1 - p)) for one element
+ fn element_loss(pred: f32, target: f32, config: BinaryCEConfig) -> f32 {
+ var p = pred;
+ if (config.from_logits) {
+ p = sigmoid(pred);
+ }
+ p = clip(p, config.epsilon);
+ return -(target * log(p) + (1.0 - target) * log(1.0 - p));
+ }
+
+ // mean (or sum) of element_loss
+ fn forward(predictions: []const f32, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ loss = loss + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ if (config.reduction == Reduction.Sum) {
+ return loss;
+ }
+ return loss / @as(f32, @floatFromInt(predictions.len));
+ }
+
+ // A batch of rows of width `width`, flattened row-major: the mean over every element
+ fn forward_batch(predictions: []const f32, width: usize, targets: []const f32, config: BinaryCEConfig) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ total = total + element_loss(predictions[i], targets[i], config);
+ i = i + 1;
+ }
+ _ = width;
+ return total / @as(f32, @floatFromInt(predictions.len));
+ }
+
+ // Per-element gradient of the (unreduced) loss:
+ // from logits: sigmoid(z) - t
+ // from probabilities: -t / p + (1 - t) / (1 - p)
+ fn gradient(predictions: []const f32, targets: []const f32, config: BinaryCEConfig, grads: []f32) -> void {
+ var i : usize = 0;
+ while (i < predictions.len) {
+ const t = targets[i];
+ if (config.from_logits) {
+ grads[i] = sigmoid(predictions[i]) - t;
+ } else {
+ const p = clip(predictions[i], config.epsilon);
+ grads[i] = -t / p + (1.0 - t) / (1.0 - p);
+ }
+ i = i + 1;
+ }
+ }
+
+ fn cfg(from_logits: bool) -> BinaryCEConfig {
+ return BinaryCEConfig{ .from_logits = from_logits, .epsilon = DEFAULT_EPSILON, .reduction = Reduction.Mean };
+ }
+
+ // The fixture: predictions (a, b, c) against targets (1, 0, 1)
+ fn loss_of(a: f32, b: f32, c: f32, from_logits: bool) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ return forward(&predictions, &targets, cfg(from_logits));
+ }
+
+ fn grad_of(a: f32, b: f32, c: f32, from_logits: bool, index: usize) -> f32 {
+ const predictions : [3]f32 = [a, b, c];
+ const targets : [3]f32 = [1.0, 0.0, 1.0];
+ var grads : [3]f32 = [9.0] ** 3;
+ gradient(&predictions, &targets, cfg(from_logits), &grads);
+ return grads[index];
+ }
+
+ fn batch_loss() -> f32 {
+ const predictions : [4]f32 = [0.8, 0.3, 0.9, 0.1];
+ const targets : [4]f32 = [1.0, 0.0, 1.0, 0.0];
+ return forward_batch(&predictions, 2, &targets, cfg(false));
+ }
+
+ fn valid_input() -> f32 {
+ return loss_of(0.8, 0.3, 0.9, false);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test sigmoid_basic_case
+ given input = 0.0
+ when result = sigmoid(input)
+ then @abs(result - 0.5) < 1e-6
+
+ test sigmoid_positive_input
+ given input = 1.0
+ when result = sigmoid(input)
+ then result > 0.5 and result < 1.0 and @abs(result - 0.7310586) < 0.000001
+
+ test sigmoid_negative_input
+ given input = -1.0
+ when result = sigmoid(input)
+ then result > 0.0 and result < 0.5 and @abs(result + sigmoid(1.0) - 1.0) < 0.000001
+
+ // -(ln 0.8 + ln 0.7 + ln 0.9) / 3 = 0.2283930
+ test forward_basic_case
+ given result = loss_of(0.8, 0.3, 0.9, false)
+ then result > 0.0 and @abs(result - 0.2283930) < 0.000001
+
+ // From logits (2, -1, 3) it is the same loss as from their sigmoids
+ test forward_from_logits_case
+ given result = loss_of(2.0, -1.0, 3.0, true)
+ when probs = loss_of(sigmoid(2.0), sigmoid(-1.0), sigmoid(3.0), false)
+ then result > 0.0 and @abs(result - probs) < 0.000001
+
+ // (-ln 0.8 - ln 0.7 - ln 0.9 - ln 0.9) / 4 = 0.1976349
+ test forward_batch_basic_case
+ given result = batch_loss()
+ then result > 0.0 and @abs(result - 0.1976349) < 0.000001
+
+ // t = 1 pulls p up (negative gradient), t = 0 pushes it down (positive)
+ test gradient_basic_case
+ given g0 = grad_of(0.8, 0.3, 0.9, false, 0)
+ when g1 = grad_of(0.8, 0.3, 0.9, false, 1)
+ then g0 < 0.0 and g1 > 0.0 and grad_of(0.8, 0.3, 0.9, false, 2) < 0.0 and @abs(g0 + 1.25) < 0.000001 and @abs(g1 - 1.0 / 0.7) < 0.000001
+
+ // From logits: sigmoid(z) - t, and it matches a finite difference of the loss
+ test gradient_from_logits_case
+ given g1 = grad_of(2.0, -1.0, 3.0, true, 1)
+ when slope = 3.0 * (loss_of(2.0, -1.0 + 0.001, 3.0, true) - loss_of(2.0, -1.0 - 0.001, 3.0, true)) / 0.002
+ then @abs(g1 - sigmoid(-1.0)) < 0.000001 and @abs(g1 - slope) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // targets[i] in {0, 1}: a perfect prediction costs (almost) nothing
+ invariant binary_ce_constraint_0
+ assert loss_of(1.0, 0.0, 1.0, false) < 0.000001
+
+ // 0 < epsilon << 1
+ invariant binary_ce_constraint_1
+ assert DEFAULT_EPSILON > 0.0 and DEFAULT_EPSILON < 0.001
+
+ // name: binary_ce -- positive for an imperfect prediction
+ invariant binary_ce_constraint_2
+ assert valid_input() > 0.0
+
+ // name: gradient -- zero where the logit already gives the target
+ invariant binary_ce_constraint_3
+ assert @abs(grad_of(30.0, -30.0, 30.0, true, 0)) < 0.000001
+
+ // "Bishop (2006) - Pattern Recognition and Machine Learning": the loss falls as p nears t
+ invariant binary_ce_constraint_4
+ assert loss_of(0.9, 0.1, 0.9, false) < loss_of(0.6, 0.4, 0.6, false)
+
+ // "Goodfellow et al. (2016) - Deep Learning": clipping keeps a certain wrong answer finite
+ invariant binary_ce_constraint_5
+ assert loss_of(0.0, 1.0, 0.0, false) < 20.0
diff --git a/apps/website/public/widgets/nn-bce/nn-bce.t27 b/apps/website/public/widgets/nn-bce/nn-bce.t27
new file mode 100644
index 0000000000..c1f2b414ae
--- /dev/null
+++ b/apps/website/public/widgets/nn-bce/nn-bce.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-bce.t27 -- binary cross-entropy: why a confident wrong answer costs so much
+; Source of truth for public/widgets/nn-bce/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-bce/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/loss/binary_crossentropy_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 14 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader moves three probabilities from 0.01 to 0.99. The loss is loss_of(a, b, c,
+; false) against the spec's targets (1, 0, 1), the bars grad_of, and the curve loss_of over the
+; first prediction; one tile feeds logits of 0 through the spec's sigmoid.
+
+; WHAT IT DOES NOT CLAIM: the spec clips each probability to [1e-7, 1 - 1e-7], so the curve is
+; finite at the ends of this window.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_bce;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-bce";
+pub const TITLE : str = "Cross-entropy: why a confident wrong answer costs so much";
+pub const DESCRIPTION : str = "Set three predicted probabilities against the answers 1, 0, 1: the loss is -log of the chance given to the truth. Drag toward a confident mistake and the cost climbs without bound.";
+pub const IMAGE_ALT : str = "Cross-entropy card: predictions 0.8, 0.3 and 0.9 against targets 1, 0, 1. A red curve shows the loss as prediction 0 runs from 0.01 to 0.99, rising steeply on the left, with the cursor at 0.8; gradient bars read -1.25, 1.43 and -1.11. Tiles read loss 0.2284, sure and right 0.0101, sure and wrong 4.6052, logits at 0 giving 0.6931.";
+pub const HOOK : str = "Predicting 0.99 where the answer is 1 costs 0.01; predicting 0.01 costs 4.6. The cost of a confident mistake is what trains a classifier.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/loss/binary_crossentropy_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 14 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_TARGETS : [3]u8 = [1, 0, 1];
+pub const K_P_LO_C : u8 = 1;
+pub const K_P_HI_C : u8 = 99;
+pub const K_X_TICKS_C : [5]u8 = [1, 25, 50, 75, 99];
+pub const K_LOSS_HI_C : u16 = 200;
+pub const K_LOSS_TICKS_C : [5]u8 = [0, 50, 100, 150, 200];
+pub const K_GRAD_LO_C : i16 = -1000;
+pub const K_GRAD_HI_C : u16 = 1000;
+pub const K_GRAD_TICKS : [5]i8 = [-10, -5, 0, 5, 10];
+pub const K_PRESET_A_C : [4]u8 = [80, 99, 10, 50];
+pub const K_PRESET_B_C : [4]u8 = [30, 1, 90, 50];
+pub const K_PRESET_C_C : [4]u8 = [90, 99, 10, 50];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["0.8, 0.3, 0.9", "Sure and right", "Sure and wrong", "Undecided"];
+pub const SAY_ASK : str = "Why does a confident wrong answer cost so much?";
+pub const SAY_PRED : str = "p";
+pub const SAY_TARGET : str = "answer";
+pub const SAY_CURVE_LABEL : str = "The loss as the first prediction moves, the others held";
+pub const SAY_GRAD_LABEL : str = "The gradient for each prediction";
+pub const SAY_LOSS : str = "loss";
+pub const SAY_SURE_RIGHT : str = "0.99 where right";
+pub const SAY_SURE_WRONG : str = "0.01 where right";
+pub const SAY_LOGITS : str = "logits 0, 0, 0";
+pub const SAY_NOTE : str = "The loss of one case is -(t log p + (1 - t) log(1 - p)). The gradient of a prediction is huge where the model is sure and wrong, and small where it is sure and right.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/loss/binary_crossentropy_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_targets_are_the_spec_fixture {
+ assert K_TARGETS[0] == 1;
+ assert K_TARGETS[1] == 0;
+ assert K_TARGETS[2] == 1;
+}
+
+test the_window_stays_inside_zero_and_one {
+ assert K_P_LO_C > 0;
+ assert K_P_HI_C < 100;
+ assert K_PRESET_B_C[1] == K_P_LO_C;
+ assert K_PRESET_A_C[1] == K_P_HI_C;
+}
diff --git a/apps/website/public/widgets/nn-bce/tool.js b/apps/website/public/widgets/nn-bce/tool.js
new file mode 100644
index 0000000000..3cc561b509
--- /dev/null
+++ b/apps/website/public/widgets/nn-bce/tool.js
@@ -0,0 +1,37 @@
+// nn-bce/tool.js -- lesson 14 of neural-networks: why a confident wrong answer costs so much. Binary
+// cross-entropy charges -log of the probability given to the true answer, so near 0 the cost climbs
+// without bound (clipped at eps so it stays finite). Every value is loss_of or grad_of of
+// gHashTag/t27 specs/ml/loss/binary_crossentropy_loss.t27, compiled to wasm (./logic.js), against the
+// spec's targets (1, 0, 1). Words are SAY_ and numbers K_ constants of specs/widgets/nn-bce.t27.
+import { gradOf, lossOf } from './logic.js'
+import { COLORS, K, W, ask, bars, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const T = K('K_TARGETS', [])
+const Q0 = centi(K('K_P_LO_C', 1)), Q1 = centi(K('K_P_HI_C', 99))
+const prob = (key, i) => ({ key, label: `${say('SAY_PRED')} ${i} (${say('SAY_TARGET')} ${T[i]})`, min: Q0, max: Q1, step: 0.01, fmt: (v) => fmt(v, 2) })
+const ctl = controls([prob('a', 0), prob('b', 1), prob('c', 2)],
+ { names: W.SAY_PRESET_NAMES, values: { a: K('K_PRESET_A_C', []).map(centi), b: K('K_PRESET_B_C', []).map(centi), c: K('K_PRESET_C_C', []).map(centi) } }, draw)
+
+const curve = canvas('nn-curve', say('SAY_CURVE_LABEL'))
+const grads = canvas('nn-bars', say('SAY_GRAD_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, curve, legend([[say('SAY_LOSS'), COLORS[6]]]), el('p', { class: 'dk-label' }, say('SAY_GRAD_LABEL')), grads, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ curve.replaceChildren()
+ const o = { w: 640, h: 300, x0: Q0, x1: Q1, y0: 0, y1: centi(K('K_LOSS_HI_C', 200)), xticks: K('K_X_TICKS_C', []).map(centi), yticks: K('K_LOSS_TICKS_C', []).map(centi), xlabel: `${say('SAY_PRED')} 0 (${say('SAY_TARGET')} ${T[0]})` }
+ const p = plot(curve, o)
+ p.line(sample((x) => lossOf(x, s.b, s.c, false), Q0, Q1, 200), COLORS[6], 3)
+ cursor(curve, p, s.a, [[lossOf(s.a, s.b, s.c, false), COLORS[6]]])
+ dragX(curve, p, o, (x) => ctl.set('a', x))
+ const g = [0, 1, 2].map((i) => gradOf(s.a, s.b, s.c, false, i))
+ bars(grads, { values: g, labels: [0, 1, 2].map((i) => `${say('SAY_PRED')} ${i}`), colors: g.map((v) => (v < 0 ? COLORS[0] : COLORS[3])), lo: centi(K('K_GRAD_LO_C', -1000)), hi: centi(K('K_GRAD_HI_C', 1000)), h: 220, ticks: K('K_GRAD_TICKS', []), fmt: (v) => fmt(v, 2) })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_LOSS'), value: fmt(lossOf(s.a, s.b, s.c, false), 4), tone: 6 },
+ { label: say('SAY_SURE_RIGHT'), value: fmt(lossOf(Q1, Q0, Q1, false), 4), tone: 0 },
+ { label: say('SAY_SURE_WRONG'), value: fmt(lossOf(Q0, Q1, Q0, false), 4), tone: 'bad' },
+ { label: say('SAY_LOGITS'), value: fmt(lossOf(0, 0, 0, true), 4), tone: 2 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-conv/card.png b/apps/website/public/widgets/nn-conv/card.png
new file mode 100644
index 0000000000..cde11e50e3
Binary files /dev/null and b/apps/website/public/widgets/nn-conv/card.png differ
diff --git a/apps/website/public/widgets/nn-conv/index.html b/apps/website/public/widgets/nn-conv/index.html
new file mode 100644
index 0000000000..b8e5e1157c
--- /dev/null
+++ b/apps/website/public/widgets/nn-conv/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Patches of a picture: a 2 x 2 window, averaged · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Patches of a picture: a 2 x 2 window, averaged
+
Tap a window on the 4 x 4 image: each output is the average of its 2 x 2 patch, two cells apart. Then switch to the gradient and see each output's blame shared out over its patch.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-conv/logic.js b/apps/website/public/widgets/nn-conv/logic.js
new file mode 100644
index 0000000000..9eceaf812d
--- /dev/null
+++ b/apps/website/public/widgets/nn-conv/logic.js
@@ -0,0 +1,44 @@
+// Generated by scripts/t27-logic.mjs from avgpool2d_layer.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: de15759fd183094a4dfdff2ea074dfaa79607bc31c789347eed1df151639e9fb
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-conv/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABGgVgAAF/YAF/AX1gAAF9YAN/f38Bf2ABfwF/AwgHAAECAwQBBAQFAXABAQEFAwEAEQYJAX8BQYCAwAALB3gHBm1lbW9yeQIAD3QyN192YWxpZF9pbnB1dAAAEHQyN19iYWNrd2FyZDRfYXQAAQ90MjdfcG9vbDJfdmFsdWUAAg50MjdfcG9vbF9jb3VudAAEDHQyN19wb29sNF9hdAAFFHQyN19jYWxjX291dHB1dF9zaXplAAYK0gYHBABBAgucAgMHfwF9AX8jgICAgABBwABrIgEkgICAgAACQEHAAEUNACABQYCAwIAAQcAA/AoAAAtBACECA30CQCACQcAARw0AQQAhAyABIQQDQAJAAkAgA0ECRg0AIANBA3RBwIDAgABqIQVBACEGIAQhBwNAIAZBAkYNAiAFIAZBAnRqKgIAQwAAgD6UIQhBACECA0ACQCACQSBHDQAgB0EIaiEHIAZBAWohBgwCCyAHIAJqIgkgCCAJKgIAkjgCACAJQQRqIgkgCCAJKgIAkjgCACACQRBqIQIMAAsLCyABIABBAnRqKgIAIQggAUHAAGokgICAgAAgCA8LIARBIGohBCADQQFqIQMMAAsLIAEgAmpBADYCACACQQRqIQIMAAsLSgIBfwF9I4CAgIAAQRBrIgAkgICAgAAgAEGAgID8ezYCDEHQgMCAAEECIABBDGoQg4CAgAAaIAAqAgwhASAAQRBqJICAgIAAIAEL0wEDB38BfQF/QQAhA0EAIAFBfmpBAXZBAWogAUECSRshBCABQQJ0IQUgAUEDdCEGA38CQAJAIAMgBEYNACACIAMgBGxBAnRqIQdBACEIIAAhCQNAIAggBEYNAkMAAAAAIQpBAiELIAkhAQNAAkAgCw0AIAcgCEECdGogCkMAAIA+lDgCACAJQQhqIQkgCEEBaiEIDAILIAogASoCAJIgAUEEaioCAJIhCiABIAVqIQEgC0F/aiELDAALCwsgBCAEbA8LIAAgBmohACADQQFqIQMMAAsLWwEBfyOAgICAAEEQayIBJICAgIAAAkACQCAAQQJHDQBB0IDAgABBAiABQQxqEIOAgIAAIQAMAQtB4IDAgABBASABQQxqEIOAgIAAIQALIAFBEGokgICAgAAgAAuaAQIDfwF9I4CAgIAAQdAAayIBJICAgIAAQQAhAgJAQcAARQ0AIAFBAEHAAPwLAAsgASEDAkADQCACQRBGDQEgAyACszgCACADQQRqIQMgAkEBaiECDAALCyABQcgAakIANwMAIAFCADcDQCABQQQgAUHAAGoQg4CAgAAaIAFBwABqIABBAnRqKgIAIQQgAUHQAGokgICAgAAgBAsVAEEAIABBfmpBAXZBAWogAEECSRsLC20BAEGAgMAAC2QAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAACAQAAAAEEAAEBBAACAQQAAgD8AAABAAABAQAAAgEAAAKBA";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function calcOutputSize(heightIn) {
+ return W.t27_calc_output_size(int(heightIn, 0, 4294967295, "calc_output_size.height_in")) >>> 0;
+}
+function pool4At(index) {
+ return W.t27_pool4_at(int(index, 0, 4294967295, "pool4_at.index"));
+}
+function poolCount(side) {
+ return W.t27_pool_count(int(side, 0, 4294967295, "pool_count.side")) >>> 0;
+}
+function pool2Value() {
+ return W.t27_pool2_value();
+}
+function backward4At(index) {
+ return W.t27_backward4_at(int(index, 0, 4294967295, "backward4_at.index"));
+}
+function validInput() {
+ return W.t27_valid_input() >>> 0;
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward"]);
+export {
+ __NOT_LOWERED__,
+ backward4At,
+ calcOutputSize,
+ pool2Value,
+ pool4At,
+ poolCount,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-conv/logic.t27 b/apps/website/public/widgets/nn-conv/logic.t27
new file mode 100644
index 0000000000..d542f5430a
--- /dev/null
+++ b/apps/website/public/widgets/nn-conv/logic.t27
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// 2D average pooling formula | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the kernel and stride were string constants typed `[]u32`, forward only
+// asserted that its input was non-empty, and the tests asserted `true` or
+// `result != undefined`. Average pooling now runs over one square H x H channel
+// with the default 2 x 2 window and stride 2 and no padding, and each test checks
+// numbers.
+
+module Avgpool2d;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_KERNEL : u32 = 2;
+ const DEFAULT_STRIDE : u32 = 2;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PoolConfig = struct {
+ kernel_size : []u32,
+ stride : []u32,
+ padding : []u32,
+ };
+
+ pub const PoolShape = struct {
+ batch : u32,
+ channels : u32,
+ height_in : u32,
+ width_in : u32,
+ height_out : u32,
+ width_out : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Output side for an input side n: (n - k) / stride + 1, and 0 when no window fits
+ fn calc_output_size(height_in: u32) -> u32 {
+ if (height_in < DEFAULT_KERNEL) {
+ return 0;
+ }
+ return (height_in - DEFAULT_KERNEL) / DEFAULT_STRIDE + 1;
+ }
+
+ // output[oy][ox] = mean of the k x k window at (oy * stride, ox * stride) of a side x side
+ // input; returns the number of outputs written
+ fn forward(input: []const f32, side: u32, output: []f32) -> u32 {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var sum : f32 = 0.0;
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ sum = sum + input[(oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx];
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * n + ox] = sum / area;
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ return n * n;
+ }
+
+ // backward: each window's gradient is shared equally by its k * k inputs
+ fn backward(grad_output: []const f32, side: u32, grad_input: []f32) -> void {
+ const n = calc_output_size(side);
+ const area : f32 = @as(f32, @floatFromInt(DEFAULT_KERNEL * DEFAULT_KERNEL));
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var oy : u32 = 0;
+ while (oy < n) {
+ var ox : u32 = 0;
+ while (ox < n) {
+ var ky : u32 = 0;
+ while (ky < DEFAULT_KERNEL) {
+ var kx : u32 = 0;
+ while (kx < DEFAULT_KERNEL) {
+ const idx = (oy * DEFAULT_STRIDE + ky) * side + ox * DEFAULT_STRIDE + kx;
+ grad_input[idx] = grad_input[idx] + grad_output[oy * n + ox] / area;
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ }
+
+ // 4 x 4 input 0..15, row-major: windows average to 2.5, 4.5, 10.5, 12.5
+ fn pool4_at(index: usize) -> f32 {
+ var input : [16]f32 = [0.0] ** 16;
+ var i : usize = 0;
+ while (i < 16) {
+ input[i] = @as(f32, @floatFromInt(i));
+ i = i + 1;
+ }
+ var output : [4]f32 = [0.0] ** 4;
+ _ = forward(&input, 4, &output);
+ return output[index];
+ }
+
+ // forward over a 2 x 2 input (1, 2, 3, 4) or a 1 x 1 input (5); returns the outputs written
+ fn pool_count(side: u32) -> u32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const one : [1]f32 = [5.0];
+ var output : [1]f32 = [-1.0];
+ if (side == 2) {
+ return forward(&four, 2, &output);
+ }
+ return forward(&one, 1, &output);
+ }
+
+ fn pool2_value() -> f32 {
+ const four : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [1]f32 = [-1.0];
+ _ = forward(&four, 2, &output);
+ return output[0];
+ }
+
+ fn backward4_at(index: usize) -> f32 {
+ const grad_output : [4]f32 = [4.0, 8.0, 12.0, 16.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, 4, &grad_input);
+ return grad_input[index];
+ }
+
+ fn valid_input() -> u32 {
+ return calc_output_size(4);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test calc_output_size_basic_case
+ given result = calc_output_size(4)
+ then result == 2
+
+ test calc_output_size_valid_input
+ given input = 10
+ when result = calc_output_size(input)
+ then result == 5
+
+ // A 1 x 1 input holds no 2 x 2 window
+ test calc_output_size_small_input
+ given input = 1
+ when result = calc_output_size(input)
+ then result == 0 and calc_output_size(2) == 1
+
+ test calc_output_size_large_input
+ given input = 1000
+ when result = calc_output_size(input)
+ then result == 500 and calc_output_size(1001) == 500
+
+ test forward_basic_case
+ given first = pool4_at(0)
+ when last = pool4_at(3)
+ then first == 2.5 and pool4_at(1) == 4.5 and pool4_at(2) == 10.5 and last == 12.5
+
+ // A 2 x 2 input is one window: the mean of 1, 2, 3, 4
+ test forward_non_empty_input
+ given count = pool_count(2)
+ when value = pool2_value()
+ then count == 1 and value == 2.5
+
+ test forward_single_element
+ given count = pool_count(1)
+ then count == 0
+
+ // Each of a window's 4 inputs gets a quarter of its gradient
+ test backward_spreads_the_gradient
+ given corner = backward4_at(0)
+ when last = backward4_at(15)
+ then corner == 1.0 and backward4_at(5) == 1.0 and backward4_at(2) == 2.0 and last == 4.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // kernel_size[i] >= 1
+ invariant avgpool2d_constraint_0
+ assert DEFAULT_KERNEL >= 1
+
+ // stride[i] >= 1
+ invariant avgpool2d_constraint_1
+ assert DEFAULT_STRIDE >= 1
+
+ // stride[i] <= kernel_size[i]: windows tile the input without gaps
+ invariant avgpool2d_constraint_2
+ assert DEFAULT_STRIDE <= DEFAULT_KERNEL
+
+ // name: avgpool_formula -- (n - k) / s + 1
+ invariant avgpool2d_constraint_3
+ assert valid_input() == 2
+
+ // "Zeiler & Fergus (2014) - Visualizing and Understanding Convolutional Networks": pooling halves each side
+ invariant avgpool2d_constraint_4
+ assert calc_output_size(8) == 4
+
+ // "He et al. (2016) - Deep Residual Learning for Image Recognition": a 7 x 7 map pools to 3 x 3
+ invariant avgpool2d_constraint_5
+ assert calc_output_size(7) == 3
diff --git a/apps/website/public/widgets/nn-conv/nn-conv.t27 b/apps/website/public/widgets/nn-conv/nn-conv.t27
new file mode 100644
index 0000000000..df845ff1df
--- /dev/null
+++ b/apps/website/public/widgets/nn-conv/nn-conv.t27
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-conv.t27 -- a window over a picture: 2 x 2 average pooling
+; Source of truth for public/widgets/nn-conv/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-conv/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/avgpool2d_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 10 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader taps a window, on the image or on the output, and toggles the gradient
+; view. Outputs are pool4_at(i), the output side calc_output_size(4) and the gradient
+; backward4_at(i) for the spec's output gradients (4, 8, 12, 16).
+
+; WHAT IT DOES NOT CLAIM: this is AVERAGE pooling with a fixed 2 x 2 window, not a convolution with
+; a learned kernel: the sliding conv2d spec is gHashTag/t27#10612, not merged when this page was
+; made.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_conv;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-conv";
+pub const TITLE : str = "Patches of a picture: a 2 x 2 window, averaged";
+pub const DESCRIPTION : str = "Tap a window on the 4 x 4 image: each output is the average of its 2 x 2 patch, two cells apart. Then switch to the gradient and see each output's blame shared out over its patch.";
+pub const IMAGE_ALT : str = "Window card: a 4 x 4 image holding 0 to 15, with the 2 x 2 patch 2, 3, 6, 7 outlined and the rest dimmed; beside it the 2 x 2 output 2.50, 4.50, 10.50, 12.50 with 4.50 outlined. Tiles read window 2 / 4, average 4.50, output 2 x 2 and gradient share 2.00.";
+pub const HOOK : str = "Pictures are read in patches. Here a 2 x 2 window steps by 2 over a 4 x 4 image and averages: 2.5, 4.5, 10.5, 12.5. A learned kernel comes next.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/avgpool2d_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 10 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SIDE : u8 = 4;
+pub const K_KERNEL : u8 = 2;
+pub const K_DEFAULT_KERNEL : u8 = 2;
+pub const K_DEFAULT_STRIDE : u8 = 2;
+pub const K_WIN_ROW : [4]u8 = [0, 0, 2, 2];
+pub const K_WIN_COL : [4]u8 = [0, 2, 0, 2];
+pub const K_PRESET_WIN : [3]u8 = [1, 0, 3];
+pub const K_PRESET_GRAD : [3]u8 = [0, 0, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["Window 2", "Window 1", "The gradient of window 4"];
+pub const SAY_ASK : str = "How does a network look at a picture: all at once, or in patches?";
+pub const SAY_SHOW_GRAD : str = "show the gradient";
+pub const SAY_IN_LABEL : str = "The 4 x 4 input: tap a patch";
+pub const SAY_GRAD_LABEL : str = "The gradient each input receives";
+pub const SAY_OUT_LABEL : str = "The 2 x 2 output: one number per patch";
+pub const SAY_WINDOW : str = "window";
+pub const SAY_AVERAGE : str = "its average";
+pub const SAY_OUT_SIDE : str = "output";
+pub const SAY_GRAD_SHARE : str = "gradient per input";
+pub const SAY_NOTE : str = "Average pooling, not yet a convolution: every window weighs its four cells equally. A convolution would weigh them with a learned 2 x 2 kernel and usually step by 1; that spec (t27#10612) is still open.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/avgpool2d_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_windows_tile_the_image {
+ assert K_SIDE == 4;
+ assert K_KERNEL == 2;
+ assert K_WIN_ROW[3] + K_KERNEL == K_SIDE;
+ assert K_WIN_COL[1] + K_KERNEL == K_SIDE;
+}
+
+test the_window_steps_by_its_own_size {
+ assert K_DEFAULT_KERNEL == K_DEFAULT_STRIDE;
+ assert K_WIN_COL[1] == K_DEFAULT_STRIDE;
+ assert K_PRESET_WIN[2] == 3;
+}
diff --git a/apps/website/public/widgets/nn-conv/tool.js b/apps/website/public/widgets/nn-conv/tool.js
new file mode 100644
index 0000000000..c4a3e6dabb
--- /dev/null
+++ b/apps/website/public/widgets/nn-conv/tool.js
@@ -0,0 +1,37 @@
+// nn-conv/tool.js -- lesson 10 of neural-networks: how a network looks at a picture in patches. The
+// window slides over a 4 x 4 image two cells at a time and each output is the AVERAGE of its 2 x 2
+// patch: average pooling, not a learned kernel (a sliding conv2d spec is gHashTag/t27#10612, not
+// merged). Every output is pool4_at and every gradient backward4_at of gHashTag/t27
+// specs/ml/layers/avgpool2d_layer.t27, compiled to wasm (./logic.js). The input is the spec's own,
+// cell i holding i. Words are SAY_ and numbers K_ constants of specs/widgets/nn-conv.t27.
+import { backward4At, calcOutputSize, pool4At } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fmt, heat, root, say, source, tiles } from '../nn-kit.js'
+
+const SIDE = K('K_SIDE', 4), KER = K('K_KERNEL', 2)
+const R0 = K('K_WIN_ROW', []), C0 = K('K_WIN_COL', [])
+const ctl = controls([
+ { key: 'win', kind: 'hidden', min: 0, max: 3 },
+ { key: 'grad', label: say('SAY_SHOW_GRAD'), kind: 'toggle' },
+], { names: W.SAY_PRESET_NAMES, values: { win: K('K_PRESET_WIN', []), grad: K('K_PRESET_GRAD', []) } }, draw)
+
+const img = canvas('nn-heat', say('SAY_IN_LABEL'))
+const out = canvas('nn-heat', say('SAY_OUT_LABEL'))
+const stat = el('div')
+const inLabel = el('p', { class: 'dk-label' })
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('div', { class: 'nn-two' }, el('div', {}, inLabel, img), el('div', {}, el('p', { class: 'dk-label' }, say('SAY_OUT_LABEL')), out)), stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+const inWin = (w, r, c) => r >= R0[w] && r < R0[w] + KER && c >= C0[w] && c < C0[w] + KER
+function draw(s) {
+ const n = calcOutputSize(SIDE)
+ inLabel.textContent = s.grad ? say('SAY_GRAD_LABEL') : say('SAY_IN_LABEL')
+ heat(img, { rows: SIDE, cols: SIDE, value: (r, c) => (s.grad ? backward4At(r * SIDE + c) : r * SIDE + c), lo: 0, hi: s.grad ? 4 : 15, w: 400, cell: 92, fmt: (v) => fmt(v, s.grad ? 2 : 0), mark: (r, c) => (inWin(s.win, r, c) ? 'on' : 'dim'), onTap: (r, c) => ctl.set('win', [0, 1, 2, 3].find((w) => inWin(w, r, c))) })
+ heat(out, { rows: n, cols: n, value: (r, c) => pool4At(r * n + c), lo: 0, hi: 15, w: 400, cell: 150, fmt: (v) => fmt(v, 2), mark: (r, c) => (r * n + c === s.win ? 'on' : 'dim'), onTap: (r, c) => ctl.set('win', r * n + c) })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_WINDOW'), value: `${s.win + 1} / ${n * n}`, tone: 1 },
+ { label: say('SAY_AVERAGE'), value: fmt(pool4At(s.win), 2), tone: 0 },
+ { label: say('SAY_OUT_SIDE'), value: `${n} x ${n}`, tone: 2 },
+ { label: say('SAY_GRAD_SHARE'), value: fmt(backward4At(R0[s.win] * SIDE + C0[s.win]), 2), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-dropout/card.png b/apps/website/public/widgets/nn-dropout/card.png
new file mode 100644
index 0000000000..a92eadc4eb
Binary files /dev/null and b/apps/website/public/widgets/nn-dropout/card.png differ
diff --git a/apps/website/public/widgets/nn-dropout/index.html b/apps/website/public/widgets/nn-dropout/index.html
new file mode 100644
index 0000000000..96c4832f45
--- /dev/null
+++ b/apps/website/public/widgets/nn-dropout/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Dropout: training a network with neurons switched off · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Dropout: training a network with neurons switched off
+
Switch between training and inference, inverted and classic dropout: units 1 and 3 are dropped, the survivors are scaled so that the layer's output keeps its expected size.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-dropout/logic.js b/apps/website/public/widgets/nn-dropout/logic.js
new file mode 100644
index 0000000000..53f88bc938
--- /dev/null
+++ b/apps/website/public/widgets/nn-dropout/logic.js
@@ -0,0 +1,28 @@
+// Generated by scripts/t27-logic.mjs from dropout_layer.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 1d8495587051334995edcb6e2d6a2dffb81528aa1a6a78f86d94bc88ccadb3ec
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-dropout/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABDgJgAn9/AX1gA39/fwF9AwMCAAEEBQFwAQEBBQMBABEGCQF/AUGAgMAACwctAwZtZW1vcnkCAA90MjdfYmFja3dhcmRfYXQAAA50MjdfZm9yd2FyZF9hdAABCscDAqYBAwF/AX0BfyOAgICAAEEgayICJICAgIAAIAJBEGpBACkCiIDAgAA3AwAgAkEAKQKAgMCAADcDCEMAAABAQwAAgD8gAEEBcRshAyACQQhqIQRBfCEAAkADQCAARQ0BIAQgA0MAAAAAIABBlIDAgABqLQAAGzgCACAAQQFqIQAgBEEEaiEEDAALCyACQQhqIAFBAnRqKgIAIQMgAkEgaiSAgICAACADC5wCAgR/AX0jgICAgABBIGsiAySAgICAACADQRBqQQApAoiAwIAANwMAIANBgYAENgAbIANBACkCgIDAgAA3AwggAUEBcSEEIANBG2ohBSAAQQFxIQBBACEBA30CQAJAAkACQCABQRBGDQACQCAARQ0AIANBCGogAWohBiAFLQAARQ0CIAFBlIDAgABqKgIAIQcCQCAERQ0AIAYgByAHkjgCAAwFCyAGIAc4AgAMBAsgAUGUgMCAAGoqAgAhByADQQhqIAFqIQYgBA0CIAYgB0MAAAA/lDgCAAwDCyADQQhqIAJBAnRqKgIAIQcgA0EgaiSAgICAACAHDwsgBkEANgIADAELIAYgBzgCAAsgBUEBaiEFIAFBBGohAQwACwsLLQEAQYCAwAALJAAAEEEAABBBAAAQQQAAEEEBAAEAAACAPwAAAEAAAEBAAACAQA==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function forwardAt(training, scaled, index) {
+ return W.t27_forward_at(training ? 1 : 0, scaled ? 1 : 0, int(index, 0, 4294967295, "forward_at.index"));
+}
+function backwardAt(scaled, index) {
+ return W.t27_backward_at(scaled ? 1 : 0, int(index, 0, 4294967295, "backward_at.index"));
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ backwardAt,
+ forwardAt
+};
diff --git a/apps/website/public/widgets/nn-dropout/logic.t27 b/apps/website/public/widgets/nn-dropout/logic.t27
new file mode 100644
index 0000000000..4a12f21501
--- /dev/null
+++ b/apps/website/public/widgets/nn-dropout/logic.t27
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Prevent co-adaptation of neurons | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the tests called `allocf32` and `allocbool`, which no spec declares, and
+// asserted `result != undefined`. Inference also scaled by (1 - p) even for
+// inverted dropout, which already scaled by 1 / (1 - p) in training, so the
+// expected activation shrank by (1 - p) between training and inference. Inference
+// is now the identity when scale_during_training is set.
+
+module Dropout;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_DROP_RATE : f32 = 0.5;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const DropoutConfig = struct {
+ p : f32,
+ inplace : bool,
+ scale_during_training : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(input: []const f32, output: []f32, mask: []bool, training: bool, config: DropoutConfig) → void
+ fn forward(input: []const f32, output: []f32, mask: []bool, training: bool, config: DropoutConfig) -> void {
+ const n = input.len;
+ var i : usize = 0;
+ while (i < n) {
+ if (training) {
+ if (mask[i]) {
+ // Neuron is kept, scale during training if configured
+ if (config.scale_during_training) {
+ output[i] = input[i] * (1.0 / (1.0 - config.p));
+ } else {
+ output[i] = input[i];
+ }
+ } else {
+ // Neuron is dropped
+ output[i] = 0.0;
+ }
+ } else {
+ if (config.scale_during_training) {
+ // Inverted dropout already scaled in training: inference is the identity
+ output[i] = input[i];
+ } else {
+ // Classic dropout: scale by the keep probability at inference
+ output[i] = input[i] * (1.0 - config.p);
+ }
+ }
+ i = i + 1;
+ }
+ }
+
+ // backward(grad_output: []const f32, mask: []const bool, grad_input: []f32, config: DropoutConfig) → void
+ fn backward(grad_output: []const f32, mask: []const bool, grad_input: []f32, config: DropoutConfig) -> void {
+ const n = grad_output.len;
+ var i : usize = 0;
+ while (i < n) {
+ if (mask[i]) {
+ // Gradient flows through kept neurons
+ if (config.scale_during_training) {
+ grad_input[i] = grad_output[i] * (1.0 / (1.0 - config.p));
+ } else {
+ grad_input[i] = grad_output[i];
+ }
+ } else {
+ // Gradient is zero for dropped neurons
+ grad_input[i] = 0.0;
+ }
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ fn cfg(scaled: bool) -> DropoutConfig {
+ return DropoutConfig{ .p = DEFAULT_DROP_RATE, .inplace = false, .scale_during_training = scaled };
+ }
+
+ // forward on input (1, 2, 3, 4) with mask (keep, drop, keep, drop)
+ fn forward_at(training: bool, scaled: bool, index: usize) -> f32 {
+ const input : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var output : [4]f32 = [9.0] ** 4;
+ var mask : [4]bool = [true, false, true, false];
+ forward(&input, &output, &mask, training, cfg(scaled));
+ return output[index];
+ }
+
+ fn backward_at(scaled: bool, index: usize) -> f32 {
+ const grad_output : [4]f32 = [1.0, 1.0, 1.0, 1.0];
+ const mask : [4]bool = [true, false, true, false];
+ var grad_input : [4]f32 = [9.0] ** 4;
+ backward(&grad_output, &mask, &grad_input, cfg(scaled));
+ return grad_input[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // p = 0.5, inverted: kept units are doubled, dropped ones are 0
+ test forward_basic_case
+ given kept = forward_at(true, true, 2)
+ when dropped = forward_at(true, true, 1)
+ then kept == 6.0 and dropped == 0.0 and forward_at(true, true, 0) == 2.0
+
+ // The expected activation is the same in training and inference
+ test forward_inference_keeps_the_expectation
+ given inverted = forward_at(false, true, 2)
+ when classic = forward_at(false, false, 2)
+ then inverted == 3.0 and classic == 1.5 and forward_at(true, false, 2) == 3.0
+
+ // The gradient flows only through kept units, with the same scale
+ test backward_basic_case
+ given kept = backward_at(true, 0)
+ when dropped = backward_at(true, 3)
+ then kept == 2.0 and dropped == 0.0 and backward_at(false, 2) == 1.0
+
+ invariant dropout_rate_is_a_probability
+ assert DEFAULT_DROP_RATE >= 0.0 and DEFAULT_DROP_RATE < 1.0
diff --git a/apps/website/public/widgets/nn-dropout/nn-dropout.t27 b/apps/website/public/widgets/nn-dropout/nn-dropout.t27
new file mode 100644
index 0000000000..dcee5951e5
--- /dev/null
+++ b/apps/website/public/widgets/nn-dropout/nn-dropout.t27
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-dropout.t27 -- dropout, inverted and classic: training with neurons switched off
+; Source of truth for public/widgets/nn-dropout/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-dropout/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/dropout_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 9 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader toggles training and inverted scaling. The rows are forward_at(false,
+; true, i) (the identity, so the input), forward_at(training, scaled, i) and, while training,
+; backward_at(scaled, i).
+
+; WHAT IT DOES NOT CLAIM: the mask is the spec's fixed one (keep, drop, keep, drop) with p 0.5, not
+; a random draw; the point is the scaling, not the sampling.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_dropout;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-dropout";
+pub const TITLE : str = "Dropout: training a network with neurons switched off";
+pub const DESCRIPTION : str = "Switch between training and inference, inverted and classic dropout: units 1 and 3 are dropped, the survivors are scaled so that the layer's output keeps its expected size.";
+pub const IMAGE_ALT : str = "Dropout card, training with inverted dropout: three rows of four cells. Input 1, 2, 3, 4; output 2, 0, 6, 0 with units 1 and 3 dimmed; gradient 2, 0, 2, 0. Tiles read mode training, a kept unit 2.00, a dropped unit 0.00 and the kept gradient 2.00.";
+pub const HOOK : str = "With p = 0.5, inverted dropout doubles the units it keeps while training (1, 2, 3, 4 becomes 2, 0, 6, 0) and leaves inference alone.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/dropout_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 9 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_UNITS : u8 = 4;
+pub const K_PRESET_TRAIN : [4]u8 = [1, 1, 0, 0];
+pub const K_PRESET_SCALED : [4]u8 = [1, 0, 1, 0];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Training, inverted", "Training, classic", "Inference, inverted", "Inference, classic"];
+pub const SAY_ASK : str = "Why would a network train with some of its neurons switched off?";
+pub const SAY_TRAINING : str = "training";
+pub const SAY_SCALED : str = "inverted (scale while training)";
+pub const SAY_GRID_LABEL : str = "Input, output and gradient of four units through dropout";
+pub const SAY_IN : str = "input";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "gradient";
+pub const SAY_UNIT : str = "unit";
+pub const SAY_MODE : str = "mode";
+pub const SAY_MODE_TRAIN : str = "training";
+pub const SAY_MODE_INFER : str = "inference";
+pub const SAY_KEPT_UNIT : str = "unit 0, kept";
+pub const SAY_DROPPED_UNIT : str = "unit 1, dropped";
+pub const SAY_GRAD_KEPT : str = "gradient, kept unit";
+pub const SAY_NOTE : str = "Dropping random units forces the rest not to lean on any one of them. Inverted dropout scales by 1 / (1 - p) while training so that inference is a plain pass; classic dropout scales by 1 - p at inference instead.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/dropout_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test four_units_four_modes {
+ assert K_UNITS == 4;
+ assert K_PRESET_TRAIN[0] == 1;
+ assert K_PRESET_TRAIN[3] == 0;
+}
+
+test every_mode_is_offered {
+ assert K_PRESET_SCALED[0] != K_PRESET_SCALED[1];
+ assert K_PRESET_SCALED[2] != K_PRESET_SCALED[3];
+ assert K_START_PRESET == 0;
+}
diff --git a/apps/website/public/widgets/nn-dropout/tool.js b/apps/website/public/widgets/nn-dropout/tool.js
new file mode 100644
index 0000000000..50c93745da
--- /dev/null
+++ b/apps/website/public/widgets/nn-dropout/tool.js
@@ -0,0 +1,33 @@
+// nn-dropout/tool.js -- lesson 9 of neural-networks: why a network trains with neurons switched off.
+// Dropout zeroes some units while training; inverted dropout scales the survivors up by 1 / (1 - p)
+// so inference needs no change, classic dropout scales at inference instead. Every value is
+// forward_at or backward_at of gHashTag/t27 specs/ml/layers/dropout_layer.t27, compiled to wasm
+// (./logic.js): input (1, 2, 3, 4), p 0.5, the spec's mask keeps units 0 and 2. Words are SAY_ and
+// numbers K_ constants of specs/widgets/nn-dropout.t27.
+import { backwardAt, forwardAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fmt, heat, root, say, source, tiles, tone } from '../nn-kit.js'
+
+const N = K('K_UNITS', 4)
+const ctl = controls([
+ { key: 'train', label: say('SAY_TRAINING'), kind: 'toggle' },
+ { key: 'scaled', label: say('SAY_SCALED'), kind: 'toggle' },
+], { names: W.SAY_PRESET_NAMES, values: { train: K('K_PRESET_TRAIN', []), scaled: K('K_PRESET_SCALED', []) } }, draw)
+
+const grid = canvas('nn-heat', say('SAY_GRID_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, grid, stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const train = !!s.train, scaled = !!s.scaled
+ // Inference with inverted dropout is the identity, so it shows the input itself.
+ const rows = [(i) => forwardAt(false, true, i), (i) => forwardAt(train, scaled, i), (i) => (train ? backwardAt(scaled, i) : 1)]
+ heat(grid, { rows: 3, cols: N, value: (r, c) => rows[r](c), color: (v) => tone(v, 8), rowLabels: [say('SAY_IN'), say('SAY_OUT'), say('SAY_GRAD')], labelWidth: 96, colLabels: [...Array(N)].map((_, i) => `${say('SAY_UNIT')} ${i}`), w: 640, cell: 120, fmt: (v) => fmt(v, 2), mark: (r, c) => (r > 0 && train && forwardAt(true, scaled, c) === 0 ? 'dim' : '') })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_MODE'), value: train ? say('SAY_MODE_TRAIN') : say('SAY_MODE_INFER'), tone: train ? 1 : 2 },
+ { label: say('SAY_KEPT_UNIT'), value: fmt(forwardAt(train, scaled, 0), 2), tone: 0 },
+ { label: say('SAY_DROPPED_UNIT'), value: fmt(forwardAt(train, scaled, 1), 2), tone: 6 },
+ { label: say('SAY_GRAD_KEPT'), value: fmt(backwardAt(scaled, 0), 2), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-elu/card.png b/apps/website/public/widgets/nn-elu/card.png
new file mode 100644
index 0000000000..0ab1ad1f0e
Binary files /dev/null and b/apps/website/public/widgets/nn-elu/card.png differ
diff --git a/apps/website/public/widgets/nn-elu/index.html b/apps/website/public/widgets/nn-elu/index.html
new file mode 100644
index 0000000000..d3cbc6ba1d
--- /dev/null
+++ b/apps/website/public/widgets/nn-elu/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+ELU: a negative side with a floor · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
ELU: a negative side with a floor
+
Drag along ELU: x above zero, alpha (e^x - 1) below. The negative side never falls under -alpha, so outputs average closer to zero, and its slope never quite dies.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-elu/logic.js b/apps/website/public/widgets/nn-elu/logic.js
new file mode 100644
index 0000000000..0a925499b2
--- /dev/null
+++ b/apps/website/public/widgets/nn-elu/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from elu_activation.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 9ce6cbd76c05367328ef91c3620675e88166d8d6a2753769497b0e483ec38a71
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-elu/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABGwVgAX8BfWAAAX1gAn19AX1gAX0BfWACfX8BfQMIBwABAQICAwQEBQFwAQEBBQMBABEGCQF/AUGAgMAACwdfBgZtZW1vcnkCAAx0MjdfYmF0Y2hfYXQAAA90MjdfdmFsaWRfaW5wdXQAARJ0MjdfbmVnYXRpdmVfaW5wdXQAAg50MjdfZGVyaXZhdGl2ZQADC3QyN19mb3J3YXJkAAQKpwYHSgIBfwF9I4CAgIAAQRBrIgEkgICAgAAgAUGAgICABDYCDCABQqelh/kLNwIEIAFBBGogAEECdGoqAgAhAiABQRBqJICAgIAAIAILBwBDAAAAQAsHAEMAAIC/CycBAX1DAACAPyECAkAgAEMAAAAAXg0AIAEgABCFgICAAJQhAgsgAgskAAJAIABDAAAAAF4NACABIAAQhYCAgABDAACAv5KUIQALIAAL1AICA38DfQJAAkAgACAAXA0AIAC8IgFBH3YhAgJAAkACQAJAAkACQAJAIAFB/////wdxIgNBz9i6lQRLDQAgA0GY5MX1A00NASADQZKrlPwDTQ0CDAQLIANBgICA/AdLDQYCQCABQQBIDQAgA0GY5MWVBEkNACAAQwAAAH+UDwsgAUF/Sg0DQwAAAAAhBCADQbTjv5YESw0HDAMLIANBgICAyANNDQFBACEDQwAAAAAhBSAAIQQMBAsgAkEBcyACayEDDAILIABDAACAP5IPCyAAQzuquD+UIAJBAnQqAoCAwIAAkvwAIQMLIAAgA7IiBEMAcjG/lJIiACAEQ46+vzWUIgWTIQQLIAAgBCAEIAQgBJQiBiAGQxVSNbuUQ4+qKj6SlJMiBpRDAAAAQCAGk5UgBZOSQwAAgD+SIQQgA0UNASAEIAMQhoCAgAAPCyAAIQQLIAQLpgIBBH8CQAJAIAAgAFwNACAAvCICQf////8HcUGAgID8B08NACACQYCAgIB4cSEDQQggAkEBdCIEZ2sgBEEYdiAEQYCAgAhJGyEEAkAgAUEASA0AAkAgAUH+ASAEa0wNAEMAAIB/IACYDwsCQCAEIAFqIgVBAEoNACACIAF0IANyvg8LAkAgBEEASg0AIAJBASAEa3RB////A3EgBUEXdHIgA3K+DwsgAUEXdCACar4PCyABQQAgBGtKDQECQCABQWkgBGtODQAgA74PC0GAgIAEQQAgBEEAShsgAkH///8DcXJBACABayIDQQEgASAEamsiASADIAFIG0F/aiIBdkEBakEBdiACaCABQR9xRkF/c3G+IACYIQALIAAPCyABQRd0IAJqvgsLEQEAQYCAwAALCAAAAD8AAAC/";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function forward(x, alpha) {
+ return W.t27_forward(x, alpha);
+}
+function derivative(x, alpha) {
+ return W.t27_derivative(x, alpha);
+}
+function negativeInput() {
+ return W.t27_negative_input();
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+function batchAt(index) {
+ return W.t27_batch_at(int(index, 0, 4294967295, "batch_at.index"));
+}
+const __NOT_LOWERED__ = Object.freeze(["forward_batch"]);
+export {
+ __NOT_LOWERED__,
+ batchAt,
+ derivative,
+ forward,
+ negativeInput,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-elu/logic.t27 b/apps/website/public/widgets/nn-elu/logic.t27
new file mode 100644
index 0000000000..9b2a8b838e
--- /dev/null
+++ b/apps/website/public/widgets/nn-elu/logic.t27
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Derivative | φ² + 1/φ² = 3 | TRINITY
+// #8980: `math::exp` named a module this spec never imports, forward_batch grew
+// an array with `+`, and every test called a two-argument function with one
+// argument and asserted `result != undefined`. Now each test checks a value.
+
+module Elu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_ALPHA : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const ELUConfig = struct {
+ alpha : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(x, alpha) = x for x > 0, alpha * (exp(x) - 1) otherwise
+ fn forward(x: f32, alpha: f32) -> f32 {
+ if (x > 0.0) {
+ return x;
+ }
+ return alpha * (exp(x) - 1.0);
+ }
+
+ // forward_batch: output[i] = forward(input[i], alpha)
+ fn forward_batch(input: []const f32, alpha: f32, output: []f32) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = forward(input[i], alpha);
+ i = i + 1;
+ }
+ }
+
+ // derivative(x, alpha) = 1 for x > 0, alpha * exp(x) otherwise
+ fn derivative(x: f32, alpha: f32) -> f32 {
+ if (x > 0.0) {
+ return 1.0;
+ }
+ return alpha * exp(x);
+ }
+
+ // The inputs the tests and invariants share: one negative, zero, one positive.
+ fn negative_input() -> f32 {
+ return -1.0;
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
+ }
+
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [-1.0, 0.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, DEFAULT_ALPHA, &output);
+ return output[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // ELU(-1) = exp(-1) - 1 = -0.6321206; ELU(2) = 2
+ test "forward_basic_case"
+ given input = negative_input()
+ when result = forward(input, DEFAULT_ALPHA)
+ then @abs(result - (-0.6321206)) < 0.000001 and forward(valid_input(), DEFAULT_ALPHA) == 2.0
+
+ // alpha scales only the negative side, and ELU(x) never goes below -alpha (in f32
+ // ELU(-20) rounds to exactly -alpha)
+ test "forward_alpha_scales_negative_side"
+ given input = negative_input()
+ when result = forward(input, 0.5)
+ then @abs(result - 0.5 * forward(input, 1.0)) < 0.000001 and forward(-20.0, 0.5) >= -0.5
+
+ test "forward_batch_basic_case"
+ given input = negative_input()
+ when result = batch_at(0)
+ then result == forward(input, DEFAULT_ALPHA) and batch_at(1) == 0.0 and batch_at(2) == 2.0
+
+ // ELU'(-1) = exp(-1) = 0.3678794; ELU'(2) = 1
+ test "derivative_basic_case"
+ given input = negative_input()
+ when result = derivative(input, DEFAULT_ALPHA)
+ then @abs(result - 0.3678794) < 0.000001 and derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+
+ test "derivative_matches_finite_difference"
+ given slope = (forward(-1.0 + 0.001, DEFAULT_ALPHA) - forward(-1.0 - 0.001, DEFAULT_ALPHA)) / 0.002
+ when result = derivative(-1.0, DEFAULT_ALPHA)
+ then @abs(result - slope) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ invariant elu_constraint_0
+ assert DEFAULT_ALPHA > 0.0
+
+ // name: elu_formula -- identity on the positive side
+ invariant elu_constraint_1
+ assert forward(valid_input(), DEFAULT_ALPHA) == valid_input()
+
+ // name: derivative -- 1 on the positive side
+ invariant elu_constraint_2
+ assert derivative(valid_input(), DEFAULT_ALPHA) == 1.0
+
+ // "Clevert et al. (2016) - Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)":
+ // ELU is continuous at 0
+ invariant elu_constraint_3
+ assert forward(0.0, DEFAULT_ALPHA) == 0.0
diff --git a/apps/website/public/widgets/nn-elu/nn-elu.t27 b/apps/website/public/widgets/nn-elu/nn-elu.t27
new file mode 100644
index 0000000000..9d38c009ac
--- /dev/null
+++ b/apps/website/public/widgets/nn-elu/nn-elu.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-elu.t27 -- ELU: a negative side with a floor at -alpha
+; Source of truth for public/widgets/nn-elu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-elu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/elu_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 6 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -5 to 3 and sets alpha from 0.1 to 2. The curve is
+; forward(x, alpha), the slope derivative(x, alpha), and the floor line is forward far to the left
+; (x = -100).
+
+; WHAT IT DOES NOT CLAIM: the floor is read as forward(-100, alpha), which is -alpha to f32
+; precision, not the limit itself.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_elu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-elu";
+pub const TITLE : str = "ELU: a negative side with a floor";
+pub const DESCRIPTION : str = "Drag along ELU: x above zero, alpha (e^x - 1) below. The negative side never falls under -alpha, so outputs average closer to zero, and its slope never quite dies.";
+pub const IMAGE_ALT : str = "ELU card at alpha 1, x = -3: the green curve runs along the diagonal for positive x and bends down toward the red dashed floor at -1 on the left; the dashed gold slope is 1 on the right and decays toward 0 on the left. Tiles read -0.9502, slope 0.0498, floor -1.0000 and the spec batch (-1, 0, 2) as -0.632 0.000 2.000.";
+pub const HOOK : str = "ELU(-3) with alpha 1 is -0.9502, already 95% of the way to its floor of -1, and its slope there is still 0.0498, not 0.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/elu_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 6 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -500;
+pub const K_X_HI_C : i16 = 300;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_Y_LO_C : i16 = -220;
+pub const K_Y_HI_C : i16 = 300;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 3];
+pub const K_Y_TICKS : [6]i8 = [-2, -1, 0, 1, 2, 3];
+pub const K_ALPHA_LO_C : u8 = 10;
+pub const K_ALPHA_HI_C : u8 = 200;
+pub const K_ALPHA_STEP_C : u8 = 5;
+pub const K_FAR_LEFT : u8 = 100;
+pub const K_PRESET_X_C : [4]i16 = [-300, -300, 150, -100];
+pub const K_PRESET_ALPHA_C : [4]u8 = [100, 50, 100, 200];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["alpha 1, x = -3", "alpha 0.5, x = -3", "alpha 1, x = 1.5", "alpha 2, x = -1"];
+pub const SAY_ASK : str = "What if the negative side had a floor instead of a cliff?";
+pub const SAY_X : str = "input x";
+pub const SAY_ALPHA : str = "alpha";
+pub const SAY_F : str = "ELU(x)";
+pub const SAY_DF : str = "slope";
+pub const SAY_FLOOR : str = "floor";
+pub const SAY_PLOT_LABEL : str = "ELU of x, its slope and its floor from -5 to 3";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_BATCH : str = "spec batch -1 0 2";
+pub const SAY_NOTE : str = "ReLU throws the negative side away; ELU keeps a soft version of it, so a unit stuck below zero still moves. alpha is how deep the floor lies.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/elu_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_floor_is_far_left {
+ assert K_FAR_LEFT == 100;
+ assert K_FAR_LEFT * 100 + K_X_LO_C > 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_ALPHA_C[3] == K_ALPHA_HI_C;
+ assert K_PRESET_ALPHA_C[1] >= K_ALPHA_LO_C;
+ assert K_Y_LO_C + 200 < 0;
+}
diff --git a/apps/website/public/widgets/nn-elu/tool.js b/apps/website/public/widgets/nn-elu/tool.js
new file mode 100644
index 0000000000..dce2852071
--- /dev/null
+++ b/apps/website/public/widgets/nn-elu/tool.js
@@ -0,0 +1,38 @@
+// nn-elu/tool.js -- lesson 6 of neural-networks: a negative side with a floor. ELU is x above zero
+// and alpha (e^x - 1) below, so it never falls under -alpha and its slope never quite dies. The
+// curve, the slope and the floor (forward far to the left) are forward and derivative of
+// gHashTag/t27 specs/ml/activation/elu_activation.t27, compiled to wasm (./logic.js). Words are SAY_
+// and numbers K_ constants of specs/widgets/nn-elu.t27.
+import { batchAt, derivative, forward } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const X0 = centi(K('K_X_LO_C', -500)), X1 = centi(K('K_X_HI_C', 300))
+const FAR = -K('K_FAR_LEFT', 100)
+const ctl = controls([
+ { key: 'x', label: say('SAY_X'), min: X0, max: X1, step: centi(K('K_X_STEP_C', 5)), fmt: (v) => fmt(v, 2) },
+ { key: 'alpha', label: say('SAY_ALPHA'), min: centi(K('K_ALPHA_LO_C', 10)), max: centi(K('K_ALPHA_HI_C', 200)), step: centi(K('K_ALPHA_STEP_C', 5)), fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { x: K('K_PRESET_X_C', []).map(centi), alpha: K('K_PRESET_ALPHA_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_F'), COLORS[0]], [say('SAY_DF'), COLORS[1], true], [say('SAY_FLOOR'), COLORS[6], true]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 340, x0: X0, x1: X1, y0: centi(K('K_Y_LO_C', -220)), y1: centi(K('K_Y_HI_C', 300)), xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_X') }
+ const p = plot(svg, o)
+ const floor = forward(FAR, s.alpha)
+ svg.append(svgEl('line', { x1: p.left, x2: 640 - p.right, y1: p.clampY(floor), y2: p.clampY(floor), stroke: COLORS[6], 'stroke-width': 2, 'stroke-dasharray': '8 6' }))
+ p.line(sample((x) => derivative(x, s.alpha), X0, X1, 320), COLORS[1], 2, { 'stroke-dasharray': '6 5' })
+ p.line(sample((x) => forward(x, s.alpha), X0, X1), COLORS[0], 3.5)
+ cursor(svg, p, s.x, [[forward(s.x, s.alpha), COLORS[0]], [derivative(s.x, s.alpha), COLORS[1]]])
+ dragX(svg, p, o, (x) => ctl.set('x', x))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_OUT'), value: fmt(forward(s.x, s.alpha), 4), tone: 0 },
+ { label: say('SAY_GRAD'), value: fmt(derivative(s.x, s.alpha), 4), tone: 1 },
+ { label: say('SAY_FLOOR'), value: fmt(floor, 4), tone: 6 },
+ { label: say('SAY_BATCH'), value: [0, 1, 2].map((i) => fmt(batchAt(i), 3)).join(' '), tone: 2 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-embedding/card.png b/apps/website/public/widgets/nn-embedding/card.png
new file mode 100644
index 0000000000..0f9ff780db
Binary files /dev/null and b/apps/website/public/widgets/nn-embedding/card.png differ
diff --git a/apps/website/public/widgets/nn-embedding/index.html b/apps/website/public/widgets/nn-embedding/index.html
new file mode 100644
index 0000000000..3a0eb09bf6
--- /dev/null
+++ b/apps/website/public/widgets/nn-embedding/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Embeddings: how a word becomes a row of numbers · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Embeddings: how a word becomes a row of numbers
+
Pick two tokens: each one copies its row out of the table. Then the gradient flows back only into the rows that were read, and a token read twice collects both.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-embedding/logic.js b/apps/website/public/widgets/nn-embedding/logic.js
new file mode 100644
index 0000000000..51b8d9ffe6
--- /dev/null
+++ b/apps/website/public/widgets/nn-embedding/logic.js
@@ -0,0 +1,36 @@
+// Generated by scripts/t27-logic.mjs from embedding_layer.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: ee2625cfeb90bf9220fdf4a9d1010af3a2327bceafd4fd6341b08b4e1af54094
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-embedding/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABEQNgAAF/YAF/AX1gA39/fwF9AwUEAAECAgQFAXABAQEFAwEAEQYJAX8BQYCAwAALB0gFBm1lbW9yeQIAD3QyN192YWxpZF9pbnB1dAAAC3QyN19pbml0X2F0AAELdDI3X2dyYWRfYXQAAg10MjdfbG9va3VwX2F0AAMKrgQEBABBCAtuAgF/AX0jgICAgABBIGsiASSAgICAACABQsDFg/CD7s/Avn83AhggAULbi6zoq4u+yz43AhAgAUKuy5ny24KU2D43AgggAULzidT1q/6+nj43AgAgASAAQQJ0aioCACECIAFBIGokgICAgAAgAgvMAQICfwF9I4CAgIAAQTBrIgMkgICAgAAgA0EQakEIakIANwMAIANBIGpCADcDACADQShqQgA3AwAgA0IANwMQIAMgATYCDCADIAA2AghBcCEBIANBCGohBAJAA0AgAUUNASADQRBqIAQoAgBBA3RqIgAgACoCACABQZCAwIAAaioCAJI4AgAgACAAKgIEIAFBlIDAgABqKgIAkjgCBCAEQQRqIQQgAUEIaiEBDAALCyADQRBqIAJBAnRqKgIAIQUgA0EwaiSAgICAACAFC+kBAgJ/AX0jgICAgABBwABrIgMkgICAgABBACEEIANBMGpBCGpBACkCmIDAgAA3AwAgAyABNgIsIAMgADYCKCADQQApApCAwIAANwMwIANCgIDAj4SAgPzBADcCICADQoCAgI2EgIDUwQA3AhggA0KAgICJhICAmMEANwIQIANCgICAgICAgMA/NwIIIANBMGohAQJAA0AgBEEIRg0BIAEgA0EIaiADQShqIARqKAIAQQN0aikCADcCACAEQQRqIQQgAUEIaiEBDAALCyADQTBqIAJBAnRqKgIAIQUgA0HAAGokgICAgAAgBQsLKQEAQYCAwAALIAAAgD8AAABAAABAQAAAgEAAAIC/AACAvwAAgL8AAIC/";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function lookupAt(a, b, index) {
+ return W.t27_lookup_at(int(a, 0, 4294967295, "lookup_at.a"), int(b, 0, 4294967295, "lookup_at.b"), int(index, 0, 4294967295, "lookup_at.index"));
+}
+function gradAt(a, b, index) {
+ return W.t27_grad_at(int(a, 0, 4294967295, "grad_at.a"), int(b, 0, 4294967295, "grad_at.b"), int(index, 0, 4294967295, "grad_at.index"));
+}
+function initAt(index) {
+ return W.t27_init_at(int(index, 0, 4294967295, "init_at.index"));
+}
+function validInput() {
+ return W.t27_valid_input() >>> 0;
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward", "init", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ gradAt,
+ initAt,
+ lookupAt,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-embedding/logic.t27 b/apps/website/public/widgets/nn-embedding/logic.t27
new file mode 100644
index 0000000000..836fe230b7
--- /dev/null
+++ b/apps/website/public/widgets/nn-embedding/logic.t27
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Embeddings are learned parameters | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward only asserted its arguments and wrote nothing, the tests called
+// undeclared `test_input_forward` / `test_input_init` and asserted
+// `result != undefined`, and the invariants asserted `true`. forward now looks each
+// token's row up in W (vocab_size x d_model, row-major), backward adds each
+// gradient row back into the row it came from, and init fills W deterministically.
+
+module Embedding;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_D_MODEL : u32 = 64;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const EmbeddingConfig = struct {
+ vocab_size : u32,
+ d_model : u32,
+ };
+
+ pub const EmbeddingWeights = struct {
+ W : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // embeddings[t * d + j] = W[token_ids[t] * d + j]: one row of W per token
+ fn forward(token_ids: []const u32, weights: EmbeddingWeights, embeddings: []f32, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ embeddings[t * d + j] = weights.W[row * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
+ }
+ }
+
+ // grad_W[token_ids[t]] += grad_embeddings[t]: a token seen twice collects both rows
+ fn backward(token_ids: []const u32, grad_embeddings: []const f32, grad_W: []f32, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ var t : usize = 0;
+ while (t < token_ids.len) {
+ const row : usize = token_ids[t];
+ var j : usize = 0;
+ while (j < d) {
+ grad_W[row * d + j] = grad_W[row * d + j] + grad_embeddings[t * d + j];
+ j = j + 1;
+ }
+ t = t + 1;
+ }
+ }
+
+ // Deterministic init in [-0.5, 0.5) / sqrt(d_model): a fixed hash of (row, column), so
+ // two runs start from the same weights
+ fn init(weights: EmbeddingWeights, config: EmbeddingConfig) -> void {
+ const d : usize = config.d_model;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d)));
+ var k : usize = 0;
+ while (k < config.vocab_size * d) {
+ const h : u32 = @as(u32, @intCast((k * 2654435761) % 1000));
+ weights.W[k] = (@as(f32, @floatFromInt(h)) / 1000.0 - 0.5) * scale;
+ k = k + 1;
+ }
+ }
+
+ // vocab 4, d_model 2, W row v = (10 v, 10 v + 1)
+ fn cfg() -> EmbeddingConfig {
+ return EmbeddingConfig{ .vocab_size = 4, .d_model = 2 };
+ }
+
+ fn lookup_at(a: u32, b: u32, index: usize) -> f32 {
+ var w : [8]f32 = [0.0, 1.0, 10.0, 11.0, 20.0, 21.0, 30.0, 31.0];
+ const ids : [2]u32 = [a, b];
+ var out : [4]f32 = [-1.0] ** 4;
+ forward(&ids, EmbeddingWeights{ .W = &w }, &out, cfg());
+ return out[index];
+ }
+
+ // grad_W[index] after backward of ids (a, b) with gradient rows (1, 2) and (3, 4)
+ fn grad_at(a: u32, b: u32, index: usize) -> f32 {
+ const ids : [2]u32 = [a, b];
+ const grads : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_w : [8]f32 = [0.0] ** 8;
+ backward(&ids, &grads, &grad_w, cfg());
+ return grad_w[index];
+ }
+
+ fn init_at(index: usize) -> f32 {
+ var w : [8]f32 = [9.0] ** 8;
+ init(EmbeddingWeights{ .W = &w }, cfg());
+ return w[index];
+ }
+
+ fn valid_input() -> u32 {
+ return cfg().vocab_size * cfg().d_model;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // ids (2, 0): rows (20, 21) then (0, 1)
+ test forward_basic_case
+ given first = lookup_at(2, 0, 0)
+ when last = lookup_at(2, 0, 3)
+ then first == 20.0 and lookup_at(2, 0, 1) == 21.0 and lookup_at(2, 0, 2) == 0.0 and last == 1.0
+
+ // Every weight is written, inside [-0.5, 0.5) / sqrt(2), and init is reproducible
+ test init_basic_case
+ given first = init_at(0)
+ when last = init_at(7)
+ then first != 9.0 and last != 9.0 and @abs(first) <= 0.36 and @abs(last) <= 0.36 and init_at(5) == init_at(5) and init_at(1) != init_at(2)
+
+ // A token seen twice collects both gradient rows; an unseen row stays 0
+ test backward_accumulates_repeated_tokens
+ given row1 = grad_at(1, 1, 2)
+ when unseen = grad_at(1, 1, 0)
+ then row1 == 4.0 and grad_at(1, 1, 3) == 6.0 and unseen == 0.0 and grad_at(1, 3, 6) == 3.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // vocab_size > 0
+ invariant embedding_constraint_0
+ assert cfg().vocab_size > 0
+
+ // d_model > 0
+ invariant embedding_constraint_1
+ assert cfg().d_model > 0 and DEFAULT_D_MODEL > 0
+
+ // W.size() == vocab_size * d_model
+ invariant embedding_constraint_2
+ assert valid_input() == 8
+
+ // token_ids must be in [0, vocab_size): the last id reads the last row
+ invariant embedding_constraint_3
+ assert lookup_at(3, 3, 1) == 31.0
+
+ // "Embedding dimension affects model capacity": each token gets d_model numbers
+ invariant embedding_constraint_4
+ assert lookup_at(1, 2, 1) == 11.0 and lookup_at(1, 2, 2) == 20.0
+
+ // "Larger vocab requires larger d_model": the init scale shrinks as 1 / sqrt(d_model)
+ invariant embedding_constraint_5
+ assert 1.0 / sqrt(@as(f32, 64.0)) < 1.0 / sqrt(@as(f32, 2.0))
+
+ // "Pre-trained embeddings transfer well": forward is a pure lookup, so loaded rows come out unchanged
+ invariant embedding_constraint_6
+ assert lookup_at(0, 1, 0) == 0.0 and lookup_at(0, 1, 2) == 10.0
diff --git a/apps/website/public/widgets/nn-embedding/nn-embedding.t27 b/apps/website/public/widgets/nn-embedding/nn-embedding.t27
new file mode 100644
index 0000000000..309aaa6fd1
--- /dev/null
+++ b/apps/website/public/widgets/nn-embedding/nn-embedding.t27
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-embedding.t27 -- the embedding table: a word becomes a row of numbers
+; Source of truth for public/widgets/nn-embedding/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-embedding/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/embedding_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 8 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader picks two tokens from four. The table is lookup_at(v, v, j) for every row
+; v, the output lookup_at(a, b, i), and the gradient table grad_at(a, b, i) for the spec's gradient
+; rows (1, 2) and (3, 4).
+
+; WHAT IT DOES NOT CLAIM: the table is the spec's 4 x 2 fixture (row v = 10 v, 10 v + 1); a real one
+; has a row per word and hundreds of columns.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_embedding;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-embedding";
+pub const TITLE : str = "Embeddings: how a word becomes a row of numbers";
+pub const DESCRIPTION : str = "Pick two tokens: each one copies its row out of the table. Then the gradient flows back only into the rows that were read, and a token read twice collects both.";
+pub const IMAGE_ALT : str = "Embedding card: token 2 read twice from a 4 x 2 table whose rows are (0, 1), (10, 11), (20, 21), (30, 31). Row 2 is outlined and the others dimmed; the output is (20, 21) twice, and the gradient table is all zero except row 2, which holds 4 and 6.";
+pub const HOOK : str = "A token is an index into a table, nothing more. Read token 2 twice and its row's gradient is (1 + 3, 2 + 4) = (4, 6); the other rows get 0.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/embedding_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 8 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_VOCAB : u8 = 4;
+pub const K_WIDTH : u8 = 2;
+pub const K_PRESET_A : [3]u8 = [2, 1, 0];
+pub const K_PRESET_B : [3]u8 = [2, 3, 2];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["The same token twice", "Tokens 1 and 3", "Tokens 0 and 2"];
+pub const SAY_ASK : str = "How does a word become numbers a network can use?";
+pub const SAY_TOKEN_NAMES : [4]str = ["token 0", "token 1", "token 2", "token 3"];
+pub const SAY_FIRST : str = "first token";
+pub const SAY_SECOND : str = "second token";
+pub const SAY_DIM : str = "dim";
+pub const SAY_TABLE_LABEL : str = "The embedding table: one row per token";
+pub const SAY_OUT_LABEL : str = "What the layer outputs";
+pub const SAY_GRAD_LABEL : str = "Where the gradient lands";
+pub const SAY_ROWS_TOUCHED : str = "rows read";
+pub const SAY_ONE_ROW : str = "one row, twice";
+pub const SAY_TWO_ROWS : str = "two rows";
+pub const SAY_VECTORS : str = "output";
+pub const SAY_GRAD_ROW : str = "gradient of {0}";
+pub const SAY_NOTE : str = "An embedding layer has no multiply at all: it copies rows. Learning changes only the rows it copied, which is why a rare word learns slowly.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/embedding_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_table_is_the_spec_fixture {
+ assert K_VOCAB == 4;
+ assert K_WIDTH == 2;
+ assert K_VOCAB * K_WIDTH == 8;
+}
+
+test the_presets_name_real_tokens {
+ assert K_PRESET_A[0] == K_PRESET_B[0];
+ assert K_PRESET_B[1] < K_VOCAB;
+ assert K_PRESET_A[1] != K_PRESET_B[1];
+}
diff --git a/apps/website/public/widgets/nn-embedding/tool.js b/apps/website/public/widgets/nn-embedding/tool.js
new file mode 100644
index 0000000000..937af8e3cc
--- /dev/null
+++ b/apps/website/public/widgets/nn-embedding/tool.js
@@ -0,0 +1,37 @@
+// nn-embedding/tool.js -- lesson 8 of neural-networks: how a word becomes numbers. An embedding is a
+// table with one row per token; looking a token up copies its row, and learning sends the gradient
+// back only to the rows that were looked up. Every cell is lookup_at or grad_at of gHashTag/t27
+// specs/ml/layers/embedding_layer.t27, compiled to wasm (./logic.js): vocab 4, width 2. Words are
+// SAY_ and numbers K_ constants of specs/widgets/nn-embedding.t27.
+import { gradAt, lookupAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fill, fmt, heat, root, say, source, tiles, tone } from '../nn-kit.js'
+
+const V = K('K_VOCAB', 4), D = K('K_WIDTH', 2)
+const tokens = W.SAY_TOKEN_NAMES ?? []
+const ctl = controls([
+ { key: 'a', label: say('SAY_FIRST'), kind: 'choice', options: tokens },
+ { key: 'b', label: say('SAY_SECOND'), kind: 'choice', options: tokens },
+], { names: W.SAY_PRESET_NAMES, values: { a: K('K_PRESET_A', []), b: K('K_PRESET_B', []) } }, draw)
+
+const table = canvas('nn-heat', say('SAY_TABLE_LABEL'))
+const out = canvas('nn-heat', say('SAY_OUT_LABEL'))
+const grad = canvas('nn-heat', say('SAY_GRAD_LABEL'))
+const stat = el('div')
+const cols = [...Array(D)].map((_, j) => `${say('SAY_DIM')} ${j}`)
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('p', { class: 'dk-label' }, say('SAY_TABLE_LABEL')), table,
+ el('div', { class: 'nn-two' }, el('div', {}, el('p', { class: 'dk-label' }, say('SAY_OUT_LABEL')), out), el('div', {}, el('p', { class: 'dk-label' }, say('SAY_GRAD_LABEL')), grad)), stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ // Row v of the table is what looking v up returns: lookup_at(v, v, j).
+ heat(table, { rows: V, cols: D, value: (r, c) => lookupAt(r, r, c), lo: 0, hi: 31, rowLabels: tokens, colLabels: cols, w: 300, cell: 104, fmt: (v) => fmt(v, 0), mark: (r) => (r === s.a || r === s.b ? 'on' : 'dim') })
+ heat(out, { rows: 2, cols: D, value: (r, c) => lookupAt(s.a, s.b, r * D + c), lo: 0, hi: 31, rowLabels: [tokens[s.a], tokens[s.b]], colLabels: cols, w: 300, cell: 104, fmt: (v) => fmt(v, 0) })
+ heat(grad, { rows: V, cols: D, value: (r, c) => gradAt(s.a, s.b, r * D + c), color: (v) => tone(v, 6), rowLabels: tokens, colLabels: cols, w: 300, cell: 104, fmt: (v) => fmt(v, 0), mark: (r) => (r === s.a || r === s.b ? 'on' : '') })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_ROWS_TOUCHED'), value: s.a === s.b ? say('SAY_ONE_ROW') : say('SAY_TWO_ROWS'), tone: 1 },
+ { label: say('SAY_VECTORS'), value: [0, 1, 2, 3].map((i) => fmt(lookupAt(s.a, s.b, i), 0)).join(' '), tone: 5 },
+ { label: fill(say('SAY_GRAD_ROW'), tokens[s.a]), value: [0, 1].map((j) => fmt(gradAt(s.a, s.b, s.a * D + j), 0)).join(', '), tone: 0 },
+ { label: fill(say('SAY_GRAD_ROW'), tokens[s.b]), value: [0, 1].map((j) => fmt(gradAt(s.a, s.b, s.b * D + j), 0)).join(', '), tone: 2 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-encoder/card.png b/apps/website/public/widgets/nn-encoder/card.png
new file mode 100644
index 0000000000..87c2017d27
Binary files /dev/null and b/apps/website/public/widgets/nn-encoder/card.png differ
diff --git a/apps/website/public/widgets/nn-encoder/index.html b/apps/website/public/widgets/nn-encoder/index.html
new file mode 100644
index 0000000000..4d811b2257
--- /dev/null
+++ b/apps/website/public/widgets/nn-encoder/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+An encoder block: what one transformer layer does · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
An encoder block: what one transformer layer does
+
Three positions through one transformer block: attention, a residual, a feed-forward net and layer norm. Post-norm ends every row at mean 0 and variance 1; pre-norm does not.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-encoder/logic.js b/apps/website/public/widgets/nn-encoder/logic.js
new file mode 100644
index 0000000000..7264e6e8d8
--- /dev/null
+++ b/apps/website/public/widgets/nn-encoder/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from encoder_block.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: e21519a3f7912ca70d35919cc1f9d2927b746f8e545c741e3772c1eb7e06561a
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-encoder/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABTQxgAAF/YAF/AX1gBH9/f38AYAZ/f39/f38AYAN/f38BfWABfwBgA39/fwBgCH9/f39/f39/AGACf38BfWACfX0BfWABfQF9YAJ9fwF9Aw4NAAECAwQFBgcBCAkKCwQFAXABAQEFAwEAEQYJAX8BQYCAwAALB18GBm1lbW9yeQIAD3QyN192YWxpZF9pbnB1dAAACnQyN19mZm5fYXQAAQx0MjdfYmxvY2tfYXQABBF0MjdfdGllX2F0dGVudGlvbgAIEHQyN19hdHRlbnRpb25fYXQACQrUGw0FAEHAAAvjAQIBfwF9I4CAgIAAQYADayIBJICAgIAAIAEgAUGAAWogAUGgAWogAUGgAmoQgoCAgAAgAUHQAmpBCGpBACkCiIDAgAA3AwAgAUEENgL8AiABQSA2AvQCIAFBCDYC7AIgAUEgNgLkAiABQQApAoCAwIAANwPQAiABIAFBoAJqNgL4AiABIAFBoAFqNgLwAiABIAFBgAFqNgLoAiABIAE2AuACQZCAwIAAIAFB4AJqQQRBCCABQbACaiABQdACahCDgICAACABQdACaiAAQQJ0aioCACECIAFBgANqJICAgIAAIAILvwEBAX9BACEEA0ACQCAEQSBHDQBBACEEA0ACQCAEQSBHDQBBACEEAkADQCAEQRBGDQEgAyAEakHNmbPuezYCACAEQQRqIQQMAAsLDwsgASAEakHNmbPuAzYCACAEQQRqIQQMAAsLIAAgBEEFbEEBakH/AXFBCXCzQwAAAD6UQwAAAL+SOAIAIAIgBEEHbEECakH/AXFBCXCzQwAAAD6UQwAAAL+SOAIAIAJBBGohAiAAQQRqIQAgBEEBaiEEDAALC7MCAwd/AX0CfyACQQJ0IQYgASgCGCEHIAEoAhAhCCABKAIIIQkgASgCACEKQQAhCwJAA0ACQCALIANHDQAgA0ECdCEMQQAhCwwCCyAJIAtBAnQiDGoqAgAhDSAAIQ4gCiEPIAIhAQJAA0AgAUUNASABQX9qIQEgDSAPKgIAIA4qAgCUkiENIA5BBGohDiAPQQRqIQ8MAAsLIAQgDGogDUMAAAAAEIqAgIAAOAIAIAogBmohCiALQQFqIQsMAAsLA0ACQAJAIAsgAkYNACAHIAtBAnQiCmoqAgAhDSAEIQ4gCCEPIAMhAQNAIAFFDQIgAUF/aiEBIA0gDyoCACAOKgIAlJIhDSAOQQRqIQ4gD0EEaiEPDAALCw8LIAUgCmogDTgCACAIIAxqIQggC0EBaiELDAALC9EHAgl/A30jgICAgABBwAVrIgMkgICAgAAgA0EEahCFgICAACADQTRqIANBtAFqIANB1AFqIANB1AJqEIKAgIAAAkBBMEUNACADQeQCakGggMCAAEEw/AoAAAsgA0EENgK4BSADQSA2ArAFIANBCDYCqAUgA0EgNgKgBSADIANB1AJqNgK0BSADIANB1AFqNgKsBSADIANBtAFqNgKkBSADIANBNGo2ApwFIANBjAVqIQQgA0HsBGohBSADQaQEaiEGIANB9ANqIQcgA0GUA2pBMGohCAJAAkAgAEEBcUUNAEEAIQACQANAIABBMEYNASADQQRqIABqQQQgA0GUA2ogAGoQhoCAgAAgAEEQaiEADAALCyADQZQDaiADQZQDaiADQZQDakEDQQJBBCAIIAYQh4CAgABBACEAAkADQAJAIABBMEcNAEEAIQkgA0HkAmohBiAHIQgMAgsgA0GUA2ogAGoiBkHgAGogA0EEaiAAaioCACAGQTBqKgIAkjgCACAAQQRqIQAMAAsLA0AgCUEDRg0CIAcgCUEEdCIAakEEIANBlANqIABqIgAQhoCAgAAgACADQZwFakEEQQggBSAEEIOAgIAAQQAhAAJAA0AgAEEQRg0BIAYgAGogCCAAaioCACAEIABqKgIAkjgCACAAQQRqIQAMAAsLIAZBEGohBiAIQRBqIQggCUEBaiEJDAALCyADQQRqIANBBGogA0EEakEDQQJBBCAIIAYQh4CAgABBACEAA0ACQCAAQTBHDQBBACEJIANBlANqIQgDQCAJQQNGDQMgA0GUA2ogCUEEdCIAaiIKQQQgByAAaiIAEIaAgIAAIAAgA0GcBWpBBEEIIAUgBBCDgICAACAJQQJ0IQtBACEAAkADQCAAQRBGDQEgCCAAaiIGIAZB4ABqKgIAIAQgAGoqAgCSOAIAIABBBGohAAwACwsgCkEEIANB5AJqIAtBAnRqEIaAgIAAIAhBEGohCCAJQQFqIQkMAAsLIANBlANqIABqIgYgA0EEaiAAaioCACAGQTBqKgIAkjgCACAAQQRqIQAMAAsLAkACQCABRQ0AIANB5AJqIAJBBHRqIQRDAAAAACEMQQAhAAJAA0AgAEEQRg0BIAwgBCAAaioCAEMAAIA+lJIhDCAAQQRqIQAMAAsLIAFBAUYNAUMAAAAAIQ1BACEAA0ACQCAAQRBHDQAgDSEMDAMLIA0gBCAAaioCACAMkyIOIA6UQwAAgD6UkiENIABBBGohAAwACwsgA0HkAmogAkECdGoqAgAhDAsgA0HABWokgICAgAAgDAsyAQF/QQAhAQJAA0AgAUEwRg0BIAAgAWogAUHQgMCAAGoqAgA4AgAgAUEEaiEBDAALCwvCAQMBfQJ/A31DAAAAACEDIAAhBCABIQUCQANAIAVFDQEgBUF/aiEFIAMgBCoCAJIhAyAEQQRqIQQMAAsLIAMgAbMiBpUhB0MAAAAAIQMgACEEIAEhBQJAA0AgBUUNASAFQX9qIQUgAyAEKgIAIAeTIgggCJSSIQMgBEEEaiEEDAALCyADIAaVQ6zFJzeSkSEDAkADQCABRQ0BIAIgACoCACAHkyADlTgCACAAQQRqIQAgAkEEaiECIAFBf2ohAQwACwsLxgQGA38BfQ5/An0CfwF9IANBAnQhCCADIANsQQJ0IQlDAACAPyAFIARuIgqzkZUhC0EAIQwgByENA0ACQAJAIAwgBEYNACAMIANsIQ4gBiAMIApsQQJ0Ig9qIRAgAiAPaiERIAEgD2ohEiAAIA9qIRNBACEUIA0hFQNAIBQgA0YNAiATIBQgBWxBAnQiFmohFyAHIBQgDmogA2xBAnRqIRhBACEZQ8rySfEhGgJAA0ACQCAZIANHDQBDAAAAACEbIBUhDyADIRwMAgsgEiAZIAVsQQJ0aiEdQQAhD0MAAAAAIR4CQANAIA8gCk8NASAeIBcgD0ECdCIcaioCACAdIBxqKgIAlJIhHiAPQQFqIQ8MAAsLIBggGUECdGogCyAelCIeOAIAIBlBAWohGSAaIB4QioCAgAAhGgwACwsCQANAIBxFDQEgDyAPKgIAIBqTEIuAgIAAIh44AgAgD0EEaiEPIBxBf2ohHCAbIB6SIRsMAAsLIBAgFmohGEEAIRcCQANAAkAgFyAKSQ0AQQAhDwNAIAggD0YNAyAVIA9qIhwgHCoCACAblTgCACAPQQRqIQ8MAAsLIBEgF0ECdCIZaiEdQwAAAAAhHkEAIQ9BACEcAkADQCAIIA9GDQEgHiAVIA9qKgIAIBuVIB0gHCAFbEECdGoqAgCUkiEeIA9BBGohDyAcQQFqIRwMAAsLIBggGWogHjgCACAXQQFqIRcMAAsLIBUgCGohFSAUQQFqIRQMAAsLDwsgDSAJaiENIAxBAWohDAwACwt0AgF/AX0jgICAgABBIGsiASSAgICAACABQRhqQgA3AwAgAUIANwMQIAFCADcDCEGAgcCAAEGIgcCAAEGQgcCAAEECQQFBASABQQhqIAFBEGoQh4CAgAAgAUEIaiAAQQJ0aioCACECIAFBIGokgICAgAAgAgulAQIBfwF9I4CAgIAAQbABayICJICAgIAAIAJBCGoQhYCAgAACQEEwRQ0AIAJBOGpBoIDAgABBMPwKAAALAkBByABFDQAgAkHoAGpBmIHAgABByAD8CgAACyACQQhqIAJBCGogAkEIakEDQQJBBCACQThqIAJB6ABqEIeAgIAAIAJB6ABqIAJBOGogABsgAUECdGoqAgAhAyACQbABaiSAgICAACADC0UBAX8CQCAAIABcDQACQCABIAFbDQAgAA8LAkAgAbwgALwiAnNBf0oNACABIAAgAkEASBsPCyABIAAgACABXRshAQsgAQvUAgIDfwN9AkACQCAAIABcDQAgALwiAUEfdiECAkACQAJAAkACQAJAAkAgAUH/////B3EiA0HP2LqVBEsNACADQZjkxfUDTQ0BIANBkquU/ANNDQIMBAsgA0GAgID8B0sNBgJAIAFBAEgNACADQZjkxZUESQ0AIABDAAAAf5QPCyABQX9KDQNDAAAAACEEIANBtOO/lgRLDQcMAwsgA0GAgIDIA00NAUEAIQNDAAAAACEFIAAhBAwECyACQQFzIAJrIQMMAgsgAEMAAIA/kg8LIABDO6q4P5QgAkECdCoC4IHAgACS/AAhAwsgACADsiIEQwByMb+UkiIAIARDjr6/NZQiBZMhBAsgACAEIAQgBCAElCIGIAZDFVI1u5RDj6oqPpKUkyIGlEMAAABAIAaTlSAFk5JDAACAP5IhBCADRQ0BIAQgAxCMgICAAA8LIAAhBAsgBAumAgEEfwJAAkAgACAAXA0AIAC8IgJB/////wdxQYCAgPwHTw0AIAJBgICAgHhxIQNBCCACQQF0IgRnayAEQRh2IARBgICACEkbIQQCQCABQQBIDQACQCABQf4BIARrTA0AQwAAgH8gAJgPCwJAIAQgAWoiBUEASg0AIAIgAXQgA3K+DwsCQCAEQQBKDQAgAkEBIARrdEH///8DcSAFQRd0ciADcr4PCyABQRd0IAJqvg8LIAFBACAEa0oNAQJAIAFBaSAEa04NACADvg8LQYCAgARBACAEQQBKGyACQf///wNxckEAIAFrIgNBASABIARqayIBIAMgAUgbQX9qIgF2QQFqQQF2IAJoIAFBH3FGQX9zcb4gAJghAAsgAA8LIAFBF3QgAmq+CwvyAQEAQYCAwAAL6AEAABBBAAAQQQAAEEEAABBBAACAPwAAAAAAAAA/AACAvwAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAgD8AAAAAAAAAPwAAgL8AAAAAAACAPwAAAL8AAABAAACAPwAAgD8AAAAAAAAAAAAAgD8AAIA/AAAAQAAAAEAAAIBAAAAgQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAAD8AAAC/";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function attentionAt(which, index) {
+ return W.t27_attention_at(int(which, 0, 255, "attention_at.which"), int(index, 0, 4294967295, "attention_at.index"));
+}
+function tieAttention(index) {
+ return W.t27_tie_attention(int(index, 0, 4294967295, "tie_attention.index"));
+}
+function blockAt(preNorm, which, index) {
+ return W.t27_block_at(preNorm ? 1 : 0, int(which, 0, 255, "block_at.which"), int(index, 0, 4294967295, "block_at.index"));
+}
+function ffnAt(index) {
+ return W.t27_ffn_at(int(index, 0, 4294967295, "ffn_at.index"));
+}
+function validInput() {
+ return W.t27_valid_input() >>> 0;
+}
+const __NOT_LOWERED__ = Object.freeze(["multi_head_attention", "feed_forward", "layer_norm", "forward", "cfg", "fixture_input", "ffn_fill"]);
+export {
+ __NOT_LOWERED__,
+ attentionAt,
+ blockAt,
+ ffnAt,
+ tieAttention,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-encoder/logic.t27 b/apps/website/public/widgets/nn-encoder/logic.t27
new file mode 100644
index 0000000000..1ced0caac4
--- /dev/null
+++ b/apps/website/public/widgets/nn-encoder/logic.t27
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Standard Transformer encoder block with residual connections | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: attention and the FFN returned their input ("in real implementation
+// ..."), forward skipped both residuals, and the tests asked only for a non-empty
+// output from undeclared `default_input()`. The block now computes the Transformer
+// encoder layer over a seq x d_model input. Attention is multi-head scaled
+// dot-product, softmax(q k^T / sqrt(d_head)) v per head, with identity projections.
+// Then come the position-wise FFN and two residual + layer-norm steps: post-norm
+// LN(x + f(x)) by default, pre-norm x + f(LN(x)) when use_pre_norm is set.
+
+module EncoderBlock;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_D_MODEL : u32 = 512;
+ const DEFAULT_N_HEADS : u32 = 8;
+ const DEFAULT_D_FF : u32 = 2048;
+ const DEFAULT_DROPOUT : f32 = 0.1;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const EncoderBlockConfig = struct {
+ d_model : u32,
+ n_heads : u32,
+ d_ff : u32,
+ dropout : f32,
+ use_pre_norm : bool,
+ };
+
+ pub const FFNWeights = struct {
+ w1 : []const f32,
+ b1 : []const f32,
+ w2 : []const f32,
+ b2 : []const f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // q, k, v and out are seq x d_model; attn_weights is n_heads x seq x seq. Head h uses
+ // columns h * d_head .. (h + 1) * d_head.
+ fn multi_head_attention(query: []const f32, key: []const f32, value: []const f32, seq: usize, n_heads: u32, d_model: u32, out: []f32, attn_weights: []f32) -> void {
+ const d_head : usize = d_model / n_heads;
+ const scale : f32 = 1.0 / sqrt(@as(f32, @floatFromInt(d_head)));
+ var h : usize = 0;
+ while (h < n_heads) {
+ var i : usize = 0;
+ while (i < seq) {
+ const row = attn_weights[(h * seq + i) * seq .. (h * seq + i + 1) * seq];
+ var max_s : f32 = -1.0e30;
+ var j : usize = 0;
+ while (j < seq) {
+ var dot : f32 = 0.0;
+ var c : usize = 0;
+ while (c < d_head) {
+ dot = dot + query[i * d_model + h * d_head + c] * key[j * d_model + h * d_head + c];
+ c = c + 1;
+ }
+ row[j] = dot * scale;
+ max_s = @max(max_s, row[j]);
+ j = j + 1;
+ }
+ var sum : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ row[j] = exp(row[j] - max_s);
+ sum = sum + row[j];
+ j = j + 1;
+ }
+ var c : usize = 0;
+ while (c < d_head) {
+ var acc : f32 = 0.0;
+ j = 0;
+ while (j < seq) {
+ acc = acc + row[j] / sum * value[j * d_model + h * d_head + c];
+ j = j + 1;
+ }
+ out[i * d_model + h * d_head + c] = acc;
+ c = c + 1;
+ }
+ j = 0;
+ while (j < seq) {
+ row[j] = row[j] / sum;
+ j = j + 1;
+ }
+ i = i + 1;
+ }
+ h = h + 1;
+ }
+ }
+
+ // out = W2 relu(W1 x + b1) + b2 for one row; hidden is d_ff floats of scratch.
+ // Dropout is the identity at inference, which is what this block computes.
+ fn feed_forward(input: []const f32, weights: FFNWeights, d_model: u32, d_ff: u32, hidden: []f32, out: []f32) -> void {
+ var f : usize = 0;
+ while (f < d_ff) {
+ var z : f32 = weights.b1[f];
+ var j : usize = 0;
+ while (j < d_model) {
+ z = z + weights.w1[f * d_model + j] * input[j];
+ j = j + 1;
+ }
+ hidden[f] = @max(z, 0.0);
+ f = f + 1;
+ }
+ var o : usize = 0;
+ while (o < d_model) {
+ var y : f32 = weights.b2[o];
+ f = 0;
+ while (f < d_ff) {
+ y = y + weights.w2[o * d_ff + f] * hidden[f];
+ f = f + 1;
+ }
+ out[o] = y;
+ o = o + 1;
+ }
+ }
+
+ // (x - mean) / sqrt(var + 1e-5), no learned scale or shift
+ fn layer_norm(x: []const f32, out: []f32) -> void {
+ var mean : f32 = 0.0;
+ var k : usize = 0;
+ while (k < x.len) {
+ mean = mean + x[k];
+ k = k + 1;
+ }
+ mean = mean / @as(f32, @floatFromInt(x.len));
+ var variance : f32 = 0.0;
+ k = 0;
+ while (k < x.len) {
+ variance = variance + (x[k] - mean) * (x[k] - mean);
+ k = k + 1;
+ }
+ variance = variance / @as(f32, @floatFromInt(x.len));
+ k = 0;
+ while (k < x.len) {
+ out[k] = (x[k] - mean) / sqrt(variance + 0.00001);
+ k = k + 1;
+ }
+ }
+
+ // input and out are seq x d_model. scratch needs 3 seq d_model + n_heads seq^2 + d_ff + d_model floats.
+ fn forward(input: []const f32, seq: usize, ffn: FFNWeights, config: EncoderBlockConfig, out: []f32, scratch: []f32) -> void {
+ const n : usize = seq * config.d_model;
+ const d : usize = config.d_model;
+ const normed = scratch[0..n];
+ const attn = scratch[n .. 2 * n];
+ const mid = scratch[2 * n .. 3 * n];
+ const weights = scratch[3 * n .. 3 * n + config.n_heads * seq * seq];
+ const rest = 3 * n + config.n_heads * seq * seq;
+ const hidden = scratch[rest .. rest + config.d_ff];
+ const row = scratch[rest + config.d_ff .. rest + config.d_ff + d];
+ var t : usize = 0;
+ if (config.use_pre_norm) {
+ while (t < seq) {
+ layer_norm(input[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ multi_head_attention(normed, normed, normed, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ mid[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(mid[t * d .. (t + 1) * d], normed[t * d .. (t + 1) * d]);
+ feed_forward(normed[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ out[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ t = t + 1;
+ }
+ return;
+ }
+ multi_head_attention(input, input, input, seq, config.n_heads, config.d_model, attn, weights);
+ var k : usize = 0;
+ while (k < n) {
+ normed[k] = input[k] + attn[k];
+ k = k + 1;
+ }
+ t = 0;
+ while (t < seq) {
+ layer_norm(normed[t * d .. (t + 1) * d], mid[t * d .. (t + 1) * d]);
+ feed_forward(mid[t * d .. (t + 1) * d], ffn, config.d_model, config.d_ff, hidden, row);
+ var c : usize = 0;
+ while (c < d) {
+ normed[t * d + c] = mid[t * d + c] + row[c];
+ c = c + 1;
+ }
+ layer_norm(normed[t * d .. (t + 1) * d], out[t * d .. (t + 1) * d]);
+ t = t + 1;
+ }
+ }
+
+ // The fixture: seq 3, d_model 4, 2 heads, d_ff 8
+ fn cfg(pre_norm: bool) -> EncoderBlockConfig {
+ return EncoderBlockConfig{ .d_model = 4, .n_heads = 2, .d_ff = 8, .dropout = DEFAULT_DROPOUT, .use_pre_norm = pre_norm };
+ }
+
+ fn fixture_input(x: []f32) -> void {
+ const v : [12]f32 = [1.0, 0.0, 0.5, -1.0, 0.0, 1.0, -0.5, 2.0, 1.0, 1.0, 0.0, 0.0];
+ var k : usize = 0;
+ while (k < 12) {
+ x[k] = v[k];
+ k = k + 1;
+ }
+ }
+
+ // which: 0 = out[index], 1 = attn_weights[index] of self-attention on the fixture
+ fn attention_at(which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var out : [12]f32 = [9.0] ** 12;
+ var weights : [18]f32 = [9.0] ** 18;
+ multi_head_attention(&x, &x, &x, 3, 2, 4, &out, &weights);
+ if (which == 0) {
+ return out[index];
+ }
+ return weights[index];
+ }
+
+ // Two keys equal: attention splits evenly and returns the mean of the values
+ fn tie_attention(index: usize) -> f32 {
+ const q : [2]f32 = [1.0, 1.0];
+ const k : [2]f32 = [2.0, 2.0];
+ const v : [2]f32 = [4.0, 10.0];
+ var out : [2]f32 = [0.0] ** 2;
+ var weights : [4]f32 = [0.0] ** 4;
+ multi_head_attention(&q, &k, &v, 2, 1, 1, &out, &weights);
+ return out[index];
+ }
+
+ fn ffn_fill(w1: []f32, b1: []f32, w2: []f32, b2: []f32) -> void {
+ var k : usize = 0;
+ while (k < 32) {
+ w1[k] = @as(f32, @floatFromInt((k * 5 + 1) % 9)) / 8.0 - 0.5;
+ w2[k] = @as(f32, @floatFromInt((k * 7 + 2) % 9)) / 8.0 - 0.5;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 8) {
+ b1[k] = 0.1;
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 4) {
+ b2[k] = -0.1;
+ k = k + 1;
+ }
+ }
+
+ // which: 0 = out[index]; 1 = mean of row `index`; 2 = variance of row `index`
+ fn block_at(pre_norm: bool, which: u8, index: usize) -> f32 {
+ var x : [12]f32 = undefined;
+ fixture_input(&x);
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ var out : [12]f32 = [9.0] ** 12;
+ var scratch : [3 * 12 + 18 + 8 + 4]f32 = undefined;
+ forward(&x, 3, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, cfg(pre_norm), &out, &scratch);
+ if (which == 0) {
+ return out[index];
+ }
+ var mean : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ mean = mean + out[index * 4 + c] / 4.0;
+ c = c + 1;
+ }
+ if (which == 1) {
+ return mean;
+ }
+ var variance : f32 = 0.0;
+ c = 0;
+ while (c < 4) {
+ variance = variance + (out[index * 4 + c] - mean) * (out[index * 4 + c] - mean) / 4.0;
+ c = c + 1;
+ }
+ return variance;
+ }
+
+ // FFN on row (1, 0, 0.5, -1) of the fixture weights; out[index]
+ fn ffn_at(index: usize) -> f32 {
+ var w1 : [32]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [32]f32 = undefined;
+ var b2 : [4]f32 = undefined;
+ ffn_fill(&w1, &b1, &w2, &b2);
+ const x : [4]f32 = [1.0, 0.0, 0.5, -1.0];
+ var hidden : [8]f32 = undefined;
+ var out : [4]f32 = [9.0] ** 4;
+ feed_forward(&x, FFNWeights{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, 4, 8, &hidden, &out);
+ return out[index];
+ }
+
+ fn valid_input() -> u32 {
+ return DEFAULT_D_MODEL / DEFAULT_N_HEADS;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // Every attention row is a distribution; equal keys give the mean of the values
+ test multi_head_attention_basic_case
+ given row_sum = attention_at(1, 0) + attention_at(1, 1) + attention_at(1, 2)
+ when tied = tie_attention(0)
+ then @abs(row_sum - 1.0) < 0.000001 and tied == 7.0 and tie_attention(1) == 7.0 and attention_at(0, 0) != 9.0
+
+ // Heads are independent: head 1's weights (rows 9..17) differ from head 0's
+ test multi_head_attention_heads_differ
+ given w0 = attention_at(1, 1)
+ when w1 = attention_at(1, 10)
+ then w0 != w1 and w0 > 0.0 and w1 > 0.0
+
+ // W2 relu(W1 x + b1) + b2 on the fixture row, worked by hand: (0.4125, ..., -0.178125)
+ test feed_forward_basic_case
+ given y0 = ffn_at(0)
+ when y3 = ffn_at(3)
+ then @abs(y0 - 0.4125) < 0.000001 and @abs(y3 + 0.178125) < 0.000001 and @abs(ffn_at(2) + 0.928125) < 0.000001
+
+ // Post-norm: every output row has mean 0 and variance 1
+ test forward_basic_case
+ given mean1 = block_at(false, 1, 1)
+ when var1 = block_at(false, 2, 1)
+ then @abs(mean1) < 0.00001 and @abs(var1 - 1.0) < 0.001 and @abs(block_at(false, 1, 2)) < 0.00001
+
+ // Pre-norm keeps the residual stream un-normalised, so it differs from post-norm
+ test forward_pre_norm_differs
+ given pre = block_at(true, 0, 0)
+ when post = block_at(false, 0, 0)
+ then pre != post and pre == pre
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // d_model % n_heads == 0
+ invariant encoder_block_constraint_0
+ assert DEFAULT_D_MODEL % DEFAULT_N_HEADS == 0 and cfg(false).d_model % cfg(false).n_heads == 0
+
+ // d_ff >= d_model
+ invariant encoder_block_constraint_1
+ assert DEFAULT_D_FF >= DEFAULT_D_MODEL and cfg(false).d_ff >= cfg(false).d_model
+
+ // name: encoder_block -- each head gets d_model / n_heads columns
+ invariant encoder_block_constraint_2
+ assert valid_input() == 64
+
+ // "Vaswani et al. (2017) - Attention Is All You Need": dropout is a probability
+ invariant encoder_block_constraint_3
+ assert DEFAULT_DROPOUT >= 0.0 and DEFAULT_DROPOUT < 1.0
+
+ // "Baevski et al. (2020) - Revisiting the Self-Attention": both norm placements are offered
+ invariant encoder_block_constraint_4
+ assert cfg(true).use_pre_norm and !cfg(false).use_pre_norm
diff --git a/apps/website/public/widgets/nn-encoder/nn-encoder.t27 b/apps/website/public/widgets/nn-encoder/nn-encoder.t27
new file mode 100644
index 0000000000..66df55b1cc
--- /dev/null
+++ b/apps/website/public/widgets/nn-encoder/nn-encoder.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-encoder.t27 -- a transformer encoder block, post-norm and pre-norm
+; Source of truth for public/widgets/nn-encoder/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-encoder/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/transformer/encoder_block.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 26 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader toggles post-norm and pre-norm. The output grid is block_at(pre, 0, i),
+; the row means and variances block_at(pre, 1 or 2, row), and the attention grids attention_at(1,
+; i): 3 positions, d_model 4, 2 heads, d_ff 8, the spec's weights.
+
+; WHAT IT DOES NOT CLAIM: one block with the spec's fixed weights, not a trained model; a
+; transformer stacks dozens.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_encoder;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-encoder";
+pub const TITLE : str = "An encoder block: what one transformer layer does";
+pub const DESCRIPTION : str = "Three positions through one transformer block: attention, a residual, a feed-forward net and layer norm. Post-norm ends every row at mean 0 and variance 1; pre-norm does not.";
+pub const IMAGE_ALT : str = "Encoder card, post-norm: a 3 x 4 output grid in pink and green, every row centred on 0. Tiles read row means 0.000 0.000 0.000 and variances 1.000 1.000 1.000. Below, two 3 x 3 attention grids of the block, head 0 row 0 reading 0.401 0.198 0.401.";
+pub const HOOK : str = "Post-norm rows come out at mean 0 and variance 1.000; pre-norm rows at variances 2.73, 3.33 and 0.96, because its last step is a residual add.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/transformer/encoder_block.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 26 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SEQ : u8 = 3;
+pub const K_D_MODEL : u8 = 4;
+pub const K_HEADS : u8 = 2;
+pub const K_PRESET_PRE : [2]u8 = [0, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [2]str = ["Post-norm", "Pre-norm"];
+pub const SAY_ASK : str = "What does one layer of a transformer actually do to its input?";
+pub const SAY_PRE_NORM : str = "pre-norm";
+pub const SAY_HEAD : str = "head";
+pub const SAY_POS : str = "pos";
+pub const SAY_DIM : str = "dim";
+pub const SAY_OUT_LABEL : str = "The block's output: 3 positions x 4 dimensions";
+pub const SAY_ATT_LABEL : str = "The block's self-attention weights";
+pub const SAY_MEANS : str = "row means";
+pub const SAY_VARS : str = "row variances";
+pub const SAY_ORDER : str = "order";
+pub const SAY_ORDER_POST : str = "attend, add, norm, FFN, add, norm";
+pub const SAY_ORDER_PRE : str = "norm, attend, add, norm, FFN, add";
+pub const SAY_ATT_ROW : str = "head 0, pos 0 reads";
+pub const SAY_NOTE : str = "The residual adds keep the input flowing through, so a deep stack can train; the norms keep the numbers in range. Where the norm sits decides what comes out.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/transformer/encoder_block.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test two_heads_split_the_model {
+ assert K_HEADS * 2 == K_D_MODEL;
+ assert K_SEQ == 3;
+}
+
+test both_orders_are_offered {
+ assert K_PRESET_PRE[0] == 0;
+ assert K_PRESET_PRE[1] == 1;
+}
diff --git a/apps/website/public/widgets/nn-encoder/tool.js b/apps/website/public/widgets/nn-encoder/tool.js
new file mode 100644
index 0000000000..556681f1cd
--- /dev/null
+++ b/apps/website/public/widgets/nn-encoder/tool.js
@@ -0,0 +1,34 @@
+// nn-encoder/tool.js -- lesson 26 of neural-networks: what one transformer block does. Self-attention
+// mixes the positions, a residual adds the input back, a two-layer feed-forward net works on each
+// position, and layer norm keeps every row at mean 0 and variance 1; pre-norm moves the norms in
+// front of each sublayer. Every value is block_at and attention_at of gHashTag/t27
+// specs/ml/transformer/encoder_block.t27, compiled to wasm (./logic.js): 3 positions, d_model 4, 2
+// heads, d_ff 8, the spec's own weights. Words are SAY_ and numbers K_ constants of specs/widgets/nn-encoder.t27.
+import { attentionAt, blockAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fmt, heat, heatTone, root, say, source, tiles, tone } from '../nn-kit.js'
+
+const N = K('K_SEQ', 3), D = K('K_D_MODEL', 4), H = K('K_HEADS', 2)
+const ctl = controls([
+ { key: 'pre', label: say('SAY_PRE_NORM'), kind: 'toggle' },
+], { names: W.SAY_PRESET_NAMES, values: { pre: K('K_PRESET_PRE', []) } }, draw)
+
+const out = canvas('nn-heat', say('SAY_OUT_LABEL'))
+const heads = [...Array(H)].map((_, h) => canvas('nn-heat', `${say('SAY_HEAD')} ${h}`))
+const stat = el('div')
+const pos = [...Array(N)].map((_, i) => `${say('SAY_POS')} ${i}`)
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('p', { class: 'dk-label' }, say('SAY_OUT_LABEL')), out, stat,
+ el('p', { class: 'dk-label' }, say('SAY_ATT_LABEL')), el('div', { class: 'nn-two' }, ...heads.map((svg, h) => el('div', {}, el('p', { class: 'nn-cap' }, `${say('SAY_HEAD')} ${h}`), svg))), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const pre = !!s.pre
+ heat(out, { rows: N, cols: D, value: (i, k) => blockAt(pre, 0, i * D + k), color: (v) => tone(v, 2), rowLabels: pos, colLabels: [...Array(D)].map((_, k) => `${say('SAY_DIM')} ${k}`), labelWidth: 70, w: 640, cell: 110, fmt: (v) => fmt(v, 3) })
+ heads.forEach((svg, h) => heat(svg, { rows: N, cols: N, value: (i, j) => attentionAt(1, (h * N + i) * N + j), color: (v) => heatTone(v), rowLabels: pos, colLabels: pos, labelWidth: 70, w: 400, cell: 100, fmt: (v) => fmt(v, 3) }))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_MEANS'), value: [0, 1, 2].map((i) => fmt(blockAt(pre, 1, i), 3)).join(' '), tone: 0 },
+ { label: say('SAY_VARS'), value: [0, 1, 2].map((i) => fmt(blockAt(pre, 2, i), 3)).join(' '), tone: 1 },
+ { label: say('SAY_ORDER'), value: pre ? say('SAY_ORDER_PRE') : say('SAY_ORDER_POST'), tone: 2 },
+ { label: say('SAY_ATT_ROW'), value: [0, 1, 2].map((j) => fmt(attentionAt(1, j), 3)).join(' '), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-gelu/card.png b/apps/website/public/widgets/nn-gelu/card.png
new file mode 100644
index 0000000000..e852eb5e71
Binary files /dev/null and b/apps/website/public/widgets/nn-gelu/card.png differ
diff --git a/apps/website/public/widgets/nn-gelu/index.html b/apps/website/public/widgets/nn-gelu/index.html
new file mode 100644
index 0000000000..1ba7192bc6
--- /dev/null
+++ b/apps/website/public/widgets/nn-gelu/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+GELU: why transformers bend a little below zero · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
GELU: why transformers bend a little below zero
+
Drag along GELU: x weighed by the chance a normal draw is below it. It dips to about -0.17 before it rises, and the tanh shortcut stays within a few thousandths of the exact form.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-gelu/logic.js b/apps/website/public/widgets/nn-gelu/logic.js
new file mode 100644
index 0000000000..09313001c9
--- /dev/null
+++ b/apps/website/public/widgets/nn-gelu/logic.js
@@ -0,0 +1,39 @@
+// Generated by scripts/t27-logic.mjs from gelu_activation.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: c17aac3c9fafca2b4b81781946ad2933b51b66dda7b62b0b79347f5cba9c8df8
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-gelu/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABEgNgAn19AX1gAn1/AX1gAX0BfQMODQABAQICAgEBAQECAgEEBQFwAQEBBQMBABEGCQF/AUGAgMAACwdrBwZtZW1vcnkCAA50MjdfcHJvZHVjdF9vZgAAD3QyN19iYWNrd2FyZF9hdAABDnQyN19mb3J3YXJkX2F0AAYRdDI3X2dlbHVfZ3JhZF9vbmUACAx0MjdfZ2VsdV9vbmUACQd0MjdfZXJmAAoKsRANBwAgACABlAs6AQF/I4CAgIAAQRBrIgIkgICAgAAgAiABQQFxIgE6AA8gACABEIKAgIAAIQAgAkEQaiSAgICAACAAC6EBAQF9AkAgAUEBcUUNACAAIAAgACAAQxMnNz2UlJSSQypCTD+UEIOAgIAAIgJDAACAP5JDAAAAP5QgACAAQ09dCT6UlEMAAIA/kiAAQwAAAD+UQwAAgD8gAiAClJOUQypCTD+UlJIPCyAAQ/MEtT+VEISAgIAAIQIgACAAIABDAAAAv5SUEIuAgIAAQ5lsIECVlCACQwAAgD+SQwAAAD+UkgvsAQMBfwF9AX8jgICAgABBEGsiASSAgICAAAJAAkAgAIsiArwiA0HVvrL4A0kNAAJAIANBgICAiQRNDQBDAAAAACAAlUMAAIA/kiECDAILQwAAgD9DAAAAQCACIAKSEIWAgIAAQwAAAECSlZMhAgwBCwJAIANB+IqL9ANLDQACQCADQYCAgARJDQAgAkMAAADAlBCFgICAACICjCACQwAAAECSlSECDAILIAEgACAAlDgCDCABQQxqIQMMAQsgAiACkhCFgICAACICIAJDAAAAQJKVIQILIAFBEGokgICAgAAgAowgAiAAvEEASBsLbgACQCAAQwAAAABdRQ0AIACMEISAgIAAjA8LQwAAgD8gACAAjJQQi4CAgABDAACAPyAAQwW6pz6UQwAAgD+SlSIAIAAgACAAIABDItyHP5RD4wC6v5KUQ+PwtT+SlEOOqZG+kpRDBnmCPpKUlJMLuAUEAX8BfQJ/A30jgICAgABBEGsiASSAgICAAAJAAkAgACAAWw0AQwAAwH8hAgwBC0MAAIC/IQIgAEMAAID/Ww0AAkACQAJAAkACQAJAAkACQCAAvCIDQf////8HcSIEQcPw1owESw0AIARBmOTF9QNNDQEgBEGSq5T8A08NAkF/IQQgA0F/TA0DIABDgHExv5IhAkPR9xc3IQVBASEEDAYLIARBgICA/AdLDQQgA0EASA0HIABDgHGxQl5FDQEgAEMAAAB/lCECDAcLIARBgICAmANJDQJDAAAAACEFQQAhBAwFCyAAQzuquD+UQwAAAD8gAJiS/AAiBLIiAkPR9xc3lCEFIAAgAkOAcTG/lJIhAgwDCyAAQ4BxMT+SIQJD0fcXtyEFDAILIARB////A0sNACABIAAgAJQ4AgwgAUEMaiEECyAAIQIMAgsgAiACIAWTIgCTIAWTIQULIAAgAEMAAAA/lCIGlCICIAIgAkMQMM86lENoiAi9kpRDAACAP5IiB0MAAEBAIAYgB5STIgaTQwAAwEAgACAGlJOVlCEGAkAgBA0AIAAgACAGlCACk5MhAgwBCyAAIAYgBZOUIAWTIAKTIQICQAJAAkACQCAEQQFqDgMAAgECCyAAIAKTQwAAAD+UQwAAAL+SIQIMAwsgAEMAAIC+XQ0BIAAgApMiACAAkkMAAIA/kiECDAILIARBF3QiA0GAgID8A2q+IQUCQCAEQThLDQBBgICA/AMgA2u+IQYCQCAEQRZLDQBDAACAPyAGkyAAIAKTkiAFlCECDAMLIAAgAiAGkpNDAACAP5IgBZQhAgwCCyAAIAKTQwAAgD+SIgAgAJJDAAAAf5QgACAFlCAEQYABRhtDAACAv5IhAgwBCyACIABDAAAAP5KTQwAAAMCUIQILIAFBEGokgICAgAAgAgs6AQF/I4CAgIAAQRBrIgIkgICAgAAgAiABQQFxIgE6AA8gACABEIeAgIAAIQAgAkEQaiSAgICAACAAC1QBAX0CQAJAIAFBAXFFDQAgACAAIAAgAEMTJzc9lJSUkkMqQkw/lBCDgICAACECDAELIABD8wS1P5UQhICAgAAhAgsgAEMAAAA/lCACQwAAgD+SlAsMACAAIAEQgoCAgAALDAAgACABEIeAgIAACwoAIAAQhICAgAAL1AICA38DfQJAAkAgACAAXA0AIAC8IgFBH3YhAgJAAkACQAJAAkACQAJAIAFB/////wdxIgNBz9i6lQRLDQAgA0GY5MX1A00NASADQZKrlPwDTQ0CDAQLIANBgICA/AdLDQYCQCABQQBIDQAgA0GY5MWVBEkNACAAQwAAAH+UDwsgAUF/Sg0DQwAAAAAhBCADQbTjv5YESw0HDAMLIANBgICAyANNDQFBACEDQwAAAAAhBSAAIQQMBAsgAkEBcyACayEDDAILIABDAACAP5IPCyAAQzuquD+UIAJBAnQqAoCAwIAAkvwAIQMLIAAgA7IiBEMAcjG/lJIiACAEQ46+vzWUIgWTIQQLIAAgBCAEIAQgBJQiBiAGQxVSNbuUQ4+qKj6SlJMiBpRDAAAAQCAGk5UgBZOSQwAAgD+SIQQgA0UNASAEIAMQjICAgAAPCyAAIQQLIAQLpgIBBH8CQAJAIAAgAFwNACAAvCICQf////8HcUGAgID8B08NACACQYCAgIB4cSEDQQggAkEBdCIEZ2sgBEEYdiAEQYCAgAhJGyEEAkAgAUEASA0AAkAgAUH+ASAEa0wNAEMAAIB/IACYDwsCQCAEIAFqIgVBAEoNACACIAF0IANyvg8LAkAgBEEASg0AIAJBASAEa3RB////A3EgBUEXdHIgA3K+DwsgAUEXdCACar4PCyABQQAgBGtKDQECQCABQWkgBGtODQAgA74PC0GAgIAEQQAgBEEAShsgAkH///8DcXJBACABayIDQQEgASAEamsiASADIAFIG0F/aiIBdkEBakEBdiACaCABQR9xRkF/c3G+IACYIQALIAAPCyABQRd0IAJqvgsLEQEAQYCAwAALCAAAAD8AAAC/";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function erf(x) {
+ return W.t27_erf(x);
+}
+function geluOne(x, approximate) {
+ return W.t27_gelu_one(x, approximate ? 1 : 0);
+}
+function geluGradOne(x, approximate) {
+ return W.t27_gelu_grad_one(x, approximate ? 1 : 0);
+}
+function forwardAt(x, approximate) {
+ return W.t27_forward_at(x, approximate ? 1 : 0);
+}
+function backwardAt(x, approximate) {
+ return W.t27_backward_at(x, approximate ? 1 : 0);
+}
+function productOf(a, b) {
+ return W.t27_product_of(a, b);
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward", "config_of"]);
+export {
+ __NOT_LOWERED__,
+ backwardAt,
+ erf,
+ forwardAt,
+ geluGradOne,
+ geluOne,
+ productOf
+};
diff --git a/apps/website/public/widgets/nn-gelu/logic.t27 b/apps/website/public/widgets/nn-gelu/logic.t27
new file mode 100644
index 0000000000..4c2b4a0e53
--- /dev/null
+++ b/apps/website/public/widgets/nn-gelu/logic.t27
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Based on cumulative distribution function | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: Zig has neither `@tanh` nor `@erf`, so this never compiled. erf is now
+// a function of this spec (Abramowitz-Stegun 7.1.26, |error| < 1.5e-7), and the
+// exact derivative divides the normal density by sqrt(2 pi), not sqrt(2) (the
+// slip was invisible at x = 0, the only point the tests checked).
+
+module Gelu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const SQRT_2_OVER_PI : f32 = 0.7978845608;
+ const SQRT_PI_OVER_2 : f32 = 1.2533141373;
+ const SQRT_2 : f32 = 1.4142135624;
+ const SQRT_2_PI : f32 = 2.5066282746;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const GELUConfig = struct {
+ approximate : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // erf(x), Abramowitz-Stegun 7.1.26: max absolute error 1.5e-7
+ fn erf(x: f32) -> f32 {
+ if (x < 0.0) {
+ return -erf(-x);
+ }
+ const t = 1.0 / (1.0 + 0.3275911 * x);
+ const poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))));
+ return 1.0 - poly * exp(-x * x);
+ }
+
+ // One element. approximate: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3)));
+ // exact: 0.5 x (1 + erf(x / sqrt(2))) = x Phi(x)
+ fn gelu_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ return 0.5 * x * (1.0 + tanh(inner));
+ }
+ return 0.5 * x * (1.0 + erf(x / SQRT_2));
+ }
+
+ // d/dx of gelu_one
+ fn gelu_grad_one(x: f32, approximate: bool) -> f32 {
+ if (approximate) {
+ const inner = SQRT_2_OVER_PI * (x + 0.044715 * x * x * x);
+ const tanh_val = tanh(inner);
+ const sech_sq = 1.0 - tanh_val * tanh_val;
+ return 0.5 * (1.0 + tanh_val) + 0.5 * x * sech_sq * SQRT_2_OVER_PI * (1.0 + 3.0 * 0.044715 * x * x);
+ }
+ // Phi(x) + x phi(x), phi(x) = exp(-x^2 / 2) / sqrt(2 pi)
+ const cdf = 0.5 * (1.0 + erf(x / SQRT_2));
+ const pdf = exp(-0.5 * x * x) / SQRT_2_PI;
+ return cdf + x * pdf;
+ }
+
+ fn forward(input: []const f32, output: []f32, config: GELUConfig) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = gelu_one(input[i], config.approximate);
+ i = i + 1;
+ }
+ }
+
+ fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: GELUConfig) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ grad_input[i] = grad_output[i] * gelu_grad_one(input[i], config.approximate);
+ i = i + 1;
+ }
+ }
+
+ // forward / backward over a one-element slice, for the tests
+ fn forward_at(x: f32, approximate: bool) -> f32 {
+ const input : [1]f32 = [x];
+ var output : [1]f32 = [0.0];
+ forward(&input, &output, GELUConfig{ .approximate = approximate });
+ return output[0];
+ }
+
+ fn backward_at(x: f32, approximate: bool) -> f32 {
+ const grad_output : [1]f32 = [1.0];
+ const input : [1]f32 = [x];
+ var grad_input : [1]f32 = [0.0];
+ backward(&grad_output, &input, &grad_input, GELUConfig{ .approximate = approximate });
+ return grad_input[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // The product goes through a function so the check runs, not only folds at compile time.
+ fn product_of(a: f32, b: f32) -> f32 {
+ return a * b;
+ }
+
+ fn config_of(approximate: bool) -> GELUConfig {
+ return GELUConfig{ .approximate = approximate };
+ }
+
+ test gelu_sqrt_constants_are_reciprocals
+ given product = product_of(SQRT_2_OVER_PI, SQRT_PI_OVER_2)
+ then @abs(product - 1.0) < 0.0000005
+
+ test gelu_sqrt_2_over_pi_squares_to_2_over_pi
+ given squared = product_of(SQRT_2_OVER_PI, SQRT_2_OVER_PI)
+ then @abs(squared - 0.63661977) < 0.0000003
+
+ test gelu_config_records_the_approximation_choice
+ given exact = config_of(false)
+ when approx = config_of(true)
+ then !exact.approximate and approx.approximate
+
+ // erf(0) = 0, erf(1) = 0.8427008, erf(-1) = -erf(1)
+ test gelu_erf_known_values
+ given at_one = erf(1.0)
+ when at_zero = erf(0.0)
+ then @abs(at_zero) < 0.0000002 and @abs(at_one - 0.8427008) < 0.0000003 and erf(-1.0) == -at_one
+
+ test gelu_forward_zero_input
+ given exact = forward_at(0.0, false)
+ when approx = forward_at(0.0, true)
+ then exact == 0.0 and approx == 0.0
+
+ test gelu_backward_zero_input_derivative
+ given exact = backward_at(0.0, false)
+ when approx = backward_at(0.0, true)
+ then @abs(exact - 0.5) < 0.0000002 and approx == 0.5
+
+ test gelu_forward_monotonic_positive_tail
+ given small = forward_at(1.0, true)
+ when large = forward_at(2.0, true)
+ then small < large
+
+ // With the cubic term, approximate GELU(1) = 0.8411920; exact GELU(1) = Phi(1) = 0.8413447.
+ // Without it, 0.5 (1 + tanh(0.7978846)) = 0.8296438 -- so this pins the term.
+ test gelu_tanh_cubic_coefficient
+ given approx = forward_at(1.0, true)
+ when exact = forward_at(1.0, false)
+ then @abs(approx - 0.8411920) < 0.000001 and @abs(exact - 0.8413447) < 0.000001
+
+ test gelu_backward_exact_derivative_at_zero
+ given result = backward_at(0.0, false)
+ then @abs(result - 0.5) < 0.0000002
+
+ test gelu_forward_approx_derivative_at_zero
+ given result = backward_at(0.0, true)
+ then result == 0.5
+
+ // The derivative at x = 1 is where the old sqrt(2) slip showed: 1.0833155 exact
+ test gelu_backward_matches_finite_difference
+ given exact_slope = (forward_at(1.001, false) - forward_at(0.999, false)) / 0.002
+ when approx_slope = (forward_at(1.001, true) - forward_at(0.999, true)) / 0.002
+ then @abs(backward_at(1.0, false) - exact_slope) < 0.002 and @abs(backward_at(1.0, true) - approx_slope) < 0.002 and @abs(backward_at(1.0, false) - 1.0833155) < 0.000002
diff --git a/apps/website/public/widgets/nn-gelu/nn-gelu.t27 b/apps/website/public/widgets/nn-gelu/nn-gelu.t27
new file mode 100644
index 0000000000..7a202a415b
--- /dev/null
+++ b/apps/website/public/widgets/nn-gelu/nn-gelu.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-gelu.t27 -- GELU, exact and tanh form: the bend transformers use
+; Source of truth for public/widgets/nn-gelu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-gelu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/gelu_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 4 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -4 to 3. The exact curve is forward_at(x, false) through the
+; spec's erf, the dashed curve forward_at(x, true), the tanh form, and the dotted curve backward_at,
+; the slope.
+
+; WHAT IT DOES NOT CLAIM: the erf of the spec is the Abramowitz-Stegun 7.1.26 fit, good to 1.5e-7,
+; so the exact curve is exact to that.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_gelu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-gelu";
+pub const TITLE : str = "GELU: why transformers bend a little below zero";
+pub const DESCRIPTION : str = "Drag along GELU: x weighed by the chance a normal draw is below it. It dips to about -0.17 before it rises, and the tanh shortcut stays within a few thousandths of the exact form.";
+pub const IMAGE_ALT : str = "GELU card at x = -0.75: the thick green exact curve and the dashed pink tanh form lie on top of each other from -4 to 3, dipping just under zero near -0.75; the dotted cyan slope crosses zero there. Tiles read exact -0.16997, tanh form -0.17004 and slopes 0.0008.";
+pub const HOOK : str = "GELU(-0.75) = -0.16997 exactly and -0.17004 by the tanh form. A smooth ReLU with a small dip, used in BERT and GPT.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/gelu_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 4 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -400;
+pub const K_X_HI_C : i16 = 300;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_Y_LO_C : i16 = -50;
+pub const K_Y_HI_C : i16 = 300;
+pub const K_X_TICKS : [8]i8 = [-4, -3, -2, -1, 0, 1, 2, 3];
+pub const K_Y_TICKS_C : [4]i16 = [0, 100, 200, 300];
+pub const K_PRESET_X_C : [4]i16 = [-75, 150, -300, 0];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["The dip, x = -0.75", "x = 1.5", "Far left, x = -3", "x = 0"];
+pub const SAY_ASK : str = "Why do transformers use a curve that bends a little below zero?";
+pub const SAY_X : str = "input x";
+pub const SAY_EXACT : str = "GELU, exact";
+pub const SAY_TANH : str = "GELU, tanh form";
+pub const SAY_DF : str = "slope";
+pub const SAY_DF_TANH : str = "slope, tanh form";
+pub const SAY_PLOT_LABEL : str = "GELU of x, exact and tanh form, and its slope from -4 to 3";
+pub const SAY_NOTE : str = "GELU(x) = x Phi(x): a positive input mostly passes, a large negative one mostly vanishes, and in between the curve dips. Unlike ReLU its slope is smooth everywhere.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/gelu_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_dip_is_in_range {
+ assert K_PRESET_X_C[0] < 0;
+ assert K_PRESET_X_C[0] > K_X_LO_C;
+ assert K_Y_LO_C < 0;
+}
+
+test the_window_is_wider_on_the_left {
+ assert K_X_LO_C + 400 == 0;
+ assert K_X_HI_C == 300;
+ assert K_X_TICKS[0] * 100 == K_X_LO_C;
+}
diff --git a/apps/website/public/widgets/nn-gelu/tool.js b/apps/website/public/widgets/nn-gelu/tool.js
new file mode 100644
index 0000000000..426667d5ac
--- /dev/null
+++ b/apps/website/public/widgets/nn-gelu/tool.js
@@ -0,0 +1,35 @@
+// nn-gelu/tool.js -- lesson 4 of neural-networks: why transformers bend below zero. GELU weighs x
+// by the chance a standard normal is below it, so it dips a little under zero before it rises. The
+// exact form (through erf), the tanh form and the slope are forward_at and backward_at of
+// gHashTag/t27 specs/ml/activation/gelu_activation.t27, compiled to wasm (./logic.js). Words are SAY_
+// and numbers K_ constants of specs/widgets/nn-gelu.t27.
+import { backwardAt, forwardAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const X0 = centi(K('K_X_LO_C', -400)), X1 = centi(K('K_X_HI_C', 300))
+const ctl = controls([
+ { key: 'x', label: say('SAY_X'), min: X0, max: X1, step: centi(K('K_X_STEP_C', 5)), fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { x: K('K_PRESET_X_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_EXACT'), COLORS[0]], [say('SAY_TANH'), COLORS[3], true], [say('SAY_DF'), COLORS[2], true]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 340, x0: X0, x1: X1, y0: centi(K('K_Y_LO_C', -50)), y1: centi(K('K_Y_HI_C', 300)), xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS_C', []).map(centi), xlabel: say('SAY_X') }
+ const p = plot(svg, o)
+ p.line(sample((x) => backwardAt(x, false), X0, X1), COLORS[2], 1.5, { 'stroke-dasharray': '3 4' })
+ p.line(sample((x) => forwardAt(x, false), X0, X1), COLORS[0], 4)
+ p.line(sample((x) => forwardAt(x, true), X0, X1), COLORS[3], 2, { 'stroke-dasharray': '7 6' })
+ cursor(svg, p, s.x, [[forwardAt(s.x, false), COLORS[0]], [backwardAt(s.x, false), COLORS[2]]])
+ dragX(svg, p, o, (x) => ctl.set('x', x))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_EXACT'), value: fmt(forwardAt(s.x, false), 5), tone: 0 },
+ { label: say('SAY_TANH'), value: fmt(forwardAt(s.x, true), 5), tone: 3 },
+ { label: say('SAY_DF'), value: fmt(backwardAt(s.x, false), 4), tone: 2 },
+ { label: say('SAY_DF_TANH'), value: fmt(backwardAt(s.x, true), 4), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-huber/card.png b/apps/website/public/widgets/nn-huber/card.png
new file mode 100644
index 0000000000..43c8d97748
Binary files /dev/null and b/apps/website/public/widgets/nn-huber/card.png differ
diff --git a/apps/website/public/widgets/nn-huber/index.html b/apps/website/public/widgets/nn-huber/index.html
new file mode 100644
index 0000000000..71f3ebe9ba
--- /dev/null
+++ b/apps/website/public/widgets/nn-huber/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Huber loss: a loss that does not panic at an outlier · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Huber loss: a loss that does not panic at an outlier
+
Drag the error along: Huber is half its square inside delta and only grows linearly outside, so one wild miss pulls on the model with a bounded force. The dashed curve is the pure square.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-huber/logic.js b/apps/website/public/widgets/nn-huber/logic.js
new file mode 100644
index 0000000000..b930f5afc5
--- /dev/null
+++ b/apps/website/public/widgets/nn-huber/logic.js
@@ -0,0 +1,27 @@
+// Generated by scripts/t27-logic.mjs from huber_loss.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: ff04a9d69fad5628d6aef1141f5d303b8da9a341a4aa9172e9bb36d79ae5c313
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-huber/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABFgNgA319fQF9YAR/f399AX1gAn19AX0DBgUAAQICAgQFAXABAQEFAwEAEQYJAX8BQYCAwAALBz0EBm1lbW9yeQIADXQyN19odWJlcl90d28AAA10MjdfaHViZXJfb25lAAMQdDI3X2VsZW1lbnRfbG9zcwAECqUCBVMBAX8jgICAgABBEGsiAySAgICAACADIAFDAACAP5I4AgwgAyAAQwAAgD+SOAIIIANBCGpBAkGAgMCAACACEIGAgIAAIQEgA0EQaiSAgICAACABC1ECAX0Bf0MAAAAAIQQgASEFAkADQCAFRQ0BIAVBf2ohBSAEIAAqAgAgAioCAJMgAxCCgICAAJIhBCACQQRqIQIgAEEEaiEADAALCyAEIAGzlQsvAQF9AkAgAIsiAiABX0UNACAAIABDAAAAP5SUDwsgASAClCABIAFDAAAAv5SUkgtAAQF/I4CAgIAAQRBrIgIkgICAgAAgAiAAOAIMIAJBDGpBAUGIgMCAACABEIGAgIAAIQAgAkEQaiSAgICAACAACwwAIAAgARCCgICAAAsLFQEAQYCAwAALDAAAgD8AAIA/AAAAAA==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function elementLoss(e, delta) {
+ return W.t27_element_loss(e, delta);
+}
+function huberOne(e, delta) {
+ return W.t27_huber_one(e, delta);
+}
+function huberTwo(e0, e1, delta) {
+ return W.t27_huber_two(e0, e1, delta);
+}
+const __NOT_LOWERED__ = Object.freeze(["forward"]);
+export {
+ __NOT_LOWERED__,
+ elementLoss,
+ huberOne,
+ huberTwo
+};
diff --git a/apps/website/public/widgets/nn-huber/logic.t27 b/apps/website/public/widgets/nn-huber/logic.t27
new file mode 100644
index 0000000000..7a872d0878
--- /dev/null
+++ b/apps/website/public/widgets/nn-huber/logic.t27
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Threshold where loss changes from quadratic to linear | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/huber_loss.tri: the port kept only
+// the first parameter and returned void, so forward computed a value and
+// threw it away, and every test asserted `result != undefined`.
+
+module HuberLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const HuberLossConfig = struct {
+ delta : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // per element e = prediction - target:
+ // |e| <= delta: 0.5 e^2 else: delta |e| - 0.5 delta^2
+ fn element_loss(e: f32, delta: f32) -> f32 {
+ const a = @abs(e);
+ if (a <= delta) {
+ return 0.5 * e * e;
+ }
+ return delta * a - 0.5 * delta * delta;
+ }
+
+ // forward(predictions, targets, delta) -> f32. The .tri gives the
+ // per-element formula only; this returns the MEAN over elements
+ // (PyTorch HuberLoss default reduction). 0 for n = 0.
+ fn forward(predictions: []const f32, targets: []const f32, delta: f32) -> f32 {
+ if (predictions.len == 0) {
+ return 0.0;
+ }
+ var sum_loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ sum_loss = sum_loss + element_loss(predictions[i] - targets[i], delta);
+ i = i + 1;
+ }
+ return sum_loss / (predictions.len as f32);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ fn huber_one(e: f32, delta: f32) -> f32 {
+ const predictions : [1]f32 = [e];
+ const targets : [1]f32 = [0.0];
+ return forward(&predictions, &targets, delta);
+ }
+
+ test quadratic_inside_delta
+ given result = huber_one(0.5, 1.0)
+ when negative = huber_one(-0.5, 1.0)
+ then @abs(result - 0.125) < 0.000001 and result == negative
+
+ // delta |e| - delta^2 / 2 = 3 - 0.5
+ test linear_outside_delta
+ given result = huber_one(3.0, 1.0)
+ when wide = huber_one(3.0, 2.0)
+ then @abs(result - 2.5) < 0.000001 and @abs(wide - 4.0) < 0.000001
+
+ // the two pieces meet with equal value and slope at |e| = delta
+ test continuous_at_delta
+ given below = huber_one(0.999, 1.0)
+ when above = huber_one(1.001, 1.0)
+ then @abs(above - below - 0.002) < 0.0001
+
+ // mean of (0.125, 2.5) over two elements
+ fn huber_two(e0: f32, e1: f32, delta: f32) -> f32 {
+ const predictions : [2]f32 = [e0 + 1.0, e1 + 1.0];
+ const targets : [2]f32 = [1.0, 1.0];
+ return forward(&predictions, &targets, delta);
+ }
+
+ test forward_mean
+ given result = huber_two(0.5, 3.0, 1.0)
+ when expected = 1.3125
+ then @abs(result - expected) < 0.000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant huber_zero_at_target
+ assert huber_one(0.0, 1.0) == 0.0
diff --git a/apps/website/public/widgets/nn-huber/nn-huber.t27 b/apps/website/public/widgets/nn-huber/nn-huber.t27
new file mode 100644
index 0000000000..2db4401c30
--- /dev/null
+++ b/apps/website/public/widgets/nn-huber/nn-huber.t27
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-huber.t27 -- the Huber loss: square inside delta, linear outside
+; Source of truth for public/widgets/nn-huber/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-huber/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/loss/huber_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 15 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the error from -4 to 4 and sets delta from 0.25 to 3. The green
+; curve is huber_one(e, delta), the dashed curve huber_one(e, 1000), which never leaves its
+; quadratic part on this window, and the tiles huber_two on the spec's pair.
+
+; WHAT IT DOES NOT CLAIM: the 30-error figure in the hook is the same two fns at e = 30, read off
+; the compiled logic, not a measurement of a model.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_huber;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-huber";
+pub const TITLE : str = "Huber loss: a loss that does not panic at an outlier";
+pub const DESCRIPTION : str = "Drag the error along: Huber is half its square inside delta and only grows linearly outside, so one wild miss pulls on the model with a bounded force. The dashed curve is the pure square.";
+pub const IMAGE_ALT : str = "Huber card at delta 1, error 3: the green Huber curve is a small parabola between the cyan dashed lines at -1 and 1 and then two straight arms; the dashed gold square climbs much faster. Tiles read Huber 2.5000, square 4.5000 and, for the spec's pair of errors (0.5, 3), 1.3125 against 2.3125.";
+pub const HOOK : str = "An error of 3 costs 2.5 under Huber with delta 1 and 4.5 squared. At 30 the gap is 29.5 against 450: why robust regression uses it.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/loss/huber_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 15 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_E_LO_C : i16 = -400;
+pub const K_E_HI_C : u16 = 400;
+pub const K_Y_HI_C : u16 = 800;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [5]u8 = [0, 2, 4, 6, 8];
+pub const K_DELTA_LO_C : u8 = 25;
+pub const K_DELTA_HI_C : u16 = 300;
+pub const K_WIDE : u16 = 1000;
+pub const K_PAIR_C : [2]u16 = [50, 300];
+pub const K_PRESET_E_C : [4]i16 = [300, 50, -300, 200];
+pub const K_PRESET_DELTA_C : [4]u16 = [100, 100, 50, 300];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Error 3, delta 1", "Error 0.5, delta 1", "Error -3, delta 0.5", "Error 2, delta 3"];
+pub const SAY_ASK : str = "Can a loss stay calm when one data point is far off?";
+pub const SAY_E : str = "error e";
+pub const SAY_DELTA : str = "delta";
+pub const SAY_HUBER : str = "Huber";
+pub const SAY_SQUARE : str = "half the square";
+pub const SAY_PLOT_LABEL : str = "Huber loss and half the squared error as the error goes from -4 to 4";
+pub const SAY_PAIR : str = "Huber of ({0}, {1})";
+pub const SAY_PAIR_SQ : str = "square of ({0}, {1})";
+pub const SAY_NOTE : str = "Inside delta Huber is the square; outside it is delta |e| - delta^2 / 2, joined so that value and slope agree. Its gradient never grows past delta.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/loss/huber_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_wide_delta_never_clips_the_window {
+ assert K_WIDE * 100 > K_E_HI_C;
+ assert K_DELTA_HI_C < K_WIDE * 100;
+}
+
+test the_pair_is_the_spec_example {
+ assert K_PAIR_C[0] == 50;
+ assert K_PAIR_C[1] == 300;
+ assert K_PRESET_DELTA_C[3] == K_DELTA_HI_C;
+}
diff --git a/apps/website/public/widgets/nn-huber/tool.js b/apps/website/public/widgets/nn-huber/tool.js
new file mode 100644
index 0000000000..54845a044c
--- /dev/null
+++ b/apps/website/public/widgets/nn-huber/tool.js
@@ -0,0 +1,40 @@
+// nn-huber/tool.js -- lesson 15 of neural-networks: a loss that does not panic at an outlier. Huber
+// is half the squared error inside delta and only linear outside it, so one wild miss pulls on the
+// model with a bounded force. Both curves and every tile are huber_one and huber_two of gHashTag/t27
+// specs/ml/loss/huber_loss.t27, compiled to wasm (./logic.js); the dashed curve is the same fn with
+// a delta so wide (K_WIDE) that it never leaves its quadratic part. Words are SAY_ and numbers K_
+// constants of specs/widgets/nn-huber.t27.
+import { huberOne, huberTwo } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const E0 = centi(K('K_E_LO_C', -400)), E1 = centi(K('K_E_HI_C', 400))
+const WIDE = K('K_WIDE', 1000)
+const ctl = controls([
+ { key: 'e', label: say('SAY_E'), min: E0, max: E1, step: 0.05, fmt: (v) => fmt(v, 2) },
+ { key: 'delta', label: say('SAY_DELTA'), min: centi(K('K_DELTA_LO_C', 25)), max: centi(K('K_DELTA_HI_C', 300)), step: 0.05, fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { e: K('K_PRESET_E_C', []).map(centi), delta: K('K_PRESET_DELTA_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_HUBER'), COLORS[0]], [say('SAY_SQUARE'), COLORS[1], true], [say('SAY_DELTA'), COLORS[2], true]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 340, x0: E0, x1: E1, y0: 0, y1: centi(K('K_Y_HI_C', 800)), xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_E') }
+ const p = plot(svg, o)
+ for (const d of [s.delta, -s.delta]) svg.append(svgEl('line', { x1: p.x(d), x2: p.x(d), y1: p.top, y2: p.clampY(-Infinity), stroke: COLORS[2], 'stroke-dasharray': '3 5', 'stroke-width': 2 }))
+ p.line(sample((e) => huberOne(e, WIDE), E0, E1), COLORS[1], 2, { 'stroke-dasharray': '7 6' })
+ p.line(sample((e) => huberOne(e, s.delta), E0, E1), COLORS[0], 4)
+ cursor(svg, p, s.e, [[huberOne(s.e, s.delta), COLORS[0]], [huberOne(s.e, WIDE), COLORS[1]]])
+ dragX(svg, p, o, (e) => ctl.set('e', e))
+ const pair = K('K_PAIR_C', []).map(centi)
+ stat.replaceChildren(tiles([
+ { label: say('SAY_HUBER'), value: fmt(huberOne(s.e, s.delta), 4), tone: 0 },
+ { label: say('SAY_SQUARE'), value: fmt(huberOne(s.e, WIDE), 4), tone: 1 },
+ { label: fill2(say('SAY_PAIR'), pair), value: fmt(huberTwo(pair[0], pair[1], s.delta), 4), tone: 2 },
+ { label: fill2(say('SAY_PAIR_SQ'), pair), value: fmt(huberTwo(pair[0], pair[1], WIDE), 4), tone: 6 },
+ ]))
+}
+const fill2 = (w, v) => String(w).replace('{0}', fmt(v[0], 1)).replace('{1}', fmt(v[1], 1))
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-kit.css b/apps/website/public/widgets/nn-kit.css
new file mode 100644
index 0000000000..b5b355a71b
--- /dev/null
+++ b/apps/website/public/widgets/nn-kit.css
@@ -0,0 +1,22 @@
+/* nn-kit.css -- the drawing the neural-networks widgets add to dsp-kit.css: the opening question,
+ a drag surface, bars, heat-map cells and node labels. Every tap target is at least 44px tall on
+ a phone; a heat-map cell you can tap is drawn at least 44px wide in the 390px layout. */
+.nn-ask { margin: 0 0 10px; padding: 10px 12px; border-left: 4px solid var(--c1); border-radius: 8px; background: color-mix(in srgb, var(--c1) 8%, var(--w-panel)); color: #fff; font: 700 0.95rem/1.4 var(--w-mono); }
+.nn-drag { touch-action: none; cursor: ew-resize; }
+.nn-cursor { stroke: #fff; stroke-width: 1.5; stroke-dasharray: 4 4; opacity: 0.7; }
+.nn-bar { filter: drop-shadow(0 0 6px rgba(0, 255, 136, 0.18)); }
+.nn-bar-value { font: 700 14px var(--w-mono); }
+.nn-cell { cursor: default; }
+.nn-cell[role="button"] { cursor: pointer; }
+.nn-cell[role="button"]:focus-visible rect { stroke: var(--c1); stroke-width: 3; }
+.nn-cell-text { font-family: var(--w-mono); font-weight: 700; pointer-events: none; }
+.nn-on rect { stroke: #fff; stroke-width: 3; }
+.nn-dim { opacity: 0.32; }
+.nn-node-text { font: 700 13px var(--w-mono); pointer-events: none; }
+.nn-edge-label { fill: var(--w-text); font: 11px var(--w-mono); paint-order: stroke; stroke: #020403; stroke-width: 3px; }
+.nn-steps { display: flex; flex-wrap: wrap; gap: 6px; margin: 6px 0; }
+.nn-steps .t27-btn { min-width: 44px; min-height: 44px; font: 700 0.9rem var(--w-mono); }
+.nn-two { display: grid; grid-template-columns: minmax(0, 1fr); gap: 8px; }
+@container (min-width: 700px) { .nn-two { grid-template-columns: repeat(2, minmax(0, 1fr)); } }
+.nn-cap { margin: 4px 0 0; color: var(--w-muted); font: 0.74rem var(--w-mono); text-align: center; }
+.dk-range { height: 44px; }
diff --git a/apps/website/public/widgets/nn-kit.js b/apps/website/public/widgets/nn-kit.js
new file mode 100644
index 0000000000..5478a330e7
--- /dev/null
+++ b/apps/website/public/widgets/nn-kit.js
@@ -0,0 +1,193 @@
+// nn-kit.js -- the drawing the neural-networks widgets share, on top of dsp-kit.js (sliders, presets,
+// the URL hash, stat tiles, plot axes): function curves you drag along, bar charts, heat-maps, the
+// diverging and heat colour scales, and node-link diagrams. It computes nothing about networks:
+// every value a widget shows comes from that widget's ./logic.js, the fn bodies of a gHashTag/t27
+// specs/ml spec compiled to wasm. Every word is a SAY_ and every range a K_ constant of the
+// widget's own spec (window.T27_WIDGET). Inputs stay in this tab and in the URL hash.
+import { COLORS, el, svgEl } from './dsp-kit.js'
+
+export * from './dsp-kit.js'
+
+const css = document.createElement('link')
+css.rel = 'stylesheet'
+css.href = new URL('./nn-kit.css', import.meta.url).href
+document.head.appendChild(css)
+
+/** A spec number written in hundredths (the widget specs hold integers only), as a number. */
+export const centi = (v) => Number(v) / 100
+
+/** A value for the page, with a fixed number of decimals and an honest minus sign. */
+export const fmt = (v, digits = 3) => (Number.isFinite(v) ? v.toFixed(digits).replace(/^-(0\.0*)$/, '$1').replace('-', '−') : String(v))
+
+const mix = (a, b, t) => a.map((x, i) => Math.round(x + (b[i] - x) * t))
+const rgb = (c) => `rgb(${c[0]},${c[1]},${c[2]})`
+const NEG = [255, 77, 109]
+const MID = [10, 18, 16]
+const POS = [0, 255, 136]
+
+/** Diverging colour: pink below zero, near black at zero, green above; m is the value at full colour. */
+export function tone(v, m = 1) {
+ const t = Math.max(-1, Math.min(1, m ? v / m : 0))
+ return rgb(t < 0 ? mix(MID, NEG, -t) : mix(MID, POS, t))
+}
+
+const HEAT = [[8, 12, 30], [56, 189, 248], [0, 255, 136], [255, 215, 0], [255, 255, 255]]
+/** Sequential colour for t in 0..1: deep blue, cyan, green, gold, white. */
+export function heatTone(t) {
+ const x = Math.max(0, Math.min(1, Number.isFinite(t) ? t : 0)) * (HEAT.length - 1)
+ const i = Math.min(HEAT.length - 2, Math.floor(x))
+ return rgb(mix(HEAT[i], HEAT[i + 1], x - i))
+}
+
+/** Black or white text, whichever reads on the fill. */
+const inkOn = (fill) => {
+ const [r, g, b] = fill.match(/\d+/g).map(Number)
+ return 0.2126 * r + 0.7152 * g + 0.0722 * b > 140 ? '#020403' : '#ffffff'
+}
+
+/**
+ * Drag along a plot: a pointer anywhere on the SVG moves the cursor to that x (in data units,
+ * clamped to the plot). The whole drawing is the target, so a finger never has to find a dot.
+ */
+export function dragX(svg, p, o, onX) {
+ svg.nnPlot = { x0: o.x0, x1: o.x1, left: p.left, right: p.right }
+ if (svg.dataset.nnDrag) return
+ svg.dataset.nnDrag = '1'
+ svg.classList.add('nn-drag')
+ const toX = (ev) => {
+ const box = svg.getBoundingClientRect()
+ const vb = svg.viewBox.baseVal
+ const sx = ((ev.clientX - box.left) / box.width) * vb.width
+ const cur = svg.nnPlot
+ const v = cur.x0 + ((sx - cur.left) / (vb.width - cur.left - cur.right)) * (cur.x1 - cur.x0)
+ return Math.max(cur.x0, Math.min(cur.x1, v))
+ }
+ let down = false
+ svg.addEventListener('pointerdown', (ev) => { down = true; svg.setPointerCapture?.(ev.pointerId); onX(toX(ev)) })
+ svg.addEventListener('pointermove', (ev) => { if (down) onX(toX(ev)) })
+ const up = () => { down = false }
+ svg.addEventListener('pointerup', up)
+ svg.addEventListener('pointercancel', up)
+}
+
+/** A vertical cursor at x with a dot on each curve value given as [y, colour]. */
+export function cursor(svg, p, x, dots) {
+ svg.append(svgEl('line', { x1: p.x(x), x2: p.x(x), y1: p.top, y2: p.clampY(-Infinity), class: 'nn-cursor' }))
+ for (const [y, c] of dots) {
+ if (!Number.isFinite(y)) continue
+ svg.append(svgEl('circle', { cx: p.x(x), cy: p.clampY(y), r: 9, fill: c, opacity: 0.25 }))
+ svg.append(svgEl('circle', { cx: p.x(x), cy: p.clampY(y), r: 5, fill: c, stroke: '#020403', 'stroke-width': 2 }))
+ }
+}
+
+/** Samples a spec function on [x0, x1] for a curve: [[x, f(x)], ...]. */
+export function sample(f, x0, x1, n = 160) {
+ const out = []
+ for (let i = 0; i <= n; i++) {
+ const x = x0 + ((x1 - x0) * i) / n
+ out.push([x, f(x)])
+ }
+ return out
+}
+
+/** A legend row: [[word, colour, dashed?], ...]. */
+export const legend = (items) => el('p', { class: 'dk-legend' }, items.map(([w, c, dashed]) => el('span', {}, el('i', { style: `background:${c};${dashed ? 'opacity:0.6' : ''}` }), w)))
+
+/**
+ * A bar chart into an SVG. o: { values, labels, colors, lo, hi, fmt, w, h, title }. Bars grow from
+ * zero, each value printed on its bar.
+ */
+export function bars(svg, o) {
+ const w = o.w ?? 640, h = o.h ?? 260, left = 44, right = 12, top = 26, bottom = 34
+ svg.replaceChildren()
+ svg.setAttribute('viewBox', `0 0 ${w} ${h}`)
+ const lo = o.lo ?? 0, hi = o.hi ?? 1
+ const py = (v) => h - bottom - ((v - lo) / (hi - lo || 1)) * (h - top - bottom)
+ svg.append(svgEl('rect', { x: left, y: top, width: w - left - right, height: h - top - bottom, class: 'dk-plotbg' }))
+ for (const t of o.ticks ?? []) {
+ svg.append(svgEl('line', { x1: left, x2: w - right, y1: py(t), y2: py(t), class: 'dk-grid' }))
+ svg.append(svgEl('text', { x: left - 6, y: py(t) + 4, class: 'dk-tick', 'text-anchor': 'end' }, o.tickFmt ? o.tickFmt(t) : t))
+ }
+ const n = o.values.length
+ const slot = (w - left - right) / n
+ const bw = Math.min(110, slot * 0.64)
+ o.values.forEach((v, i) => {
+ const c = o.colors?.[i] ?? COLORS[i % COLORS.length]
+ const x = left + slot * i + (slot - bw) / 2
+ const y0 = py(Math.max(lo, Math.min(hi, 0))), y1 = py(Math.max(lo, Math.min(hi, v)))
+ svg.append(svgEl('rect', { x, y: Math.min(y0, y1), width: bw, height: Math.max(1, Math.abs(y1 - y0)), rx: 6, fill: c, class: 'nn-bar' }))
+ svg.append(svgEl('text', { x: x + bw / 2, y: Math.min(y0, y1) - 7, class: 'nn-bar-value', 'text-anchor': 'middle', fill: c }, o.fmt ? o.fmt(v) : fmt(v)))
+ svg.append(svgEl('text', { x: x + bw / 2, y: h - 12, class: 'dk-tick', 'text-anchor': 'middle' }, o.labels?.[i] ?? i))
+ })
+}
+
+/**
+ * A heat-map into an SVG. o: { rows, cols, value(r, c), color(v) (default heatTone over lo..hi),
+ * lo, hi, fmt, rowLabels, colLabels, w, cell, cellH (row height, default cell), mark(r, c) -> 'on' |
+ * 'dim' | '', onTap(r, c) }.
+ * Each cell prints its value; a tapped cell calls onTap. Returns the cell size.
+ */
+export function heat(svg, o) {
+ const left = o.rowLabels ? (o.labelWidth ?? 74) : 6, top = o.colLabels ? 26 : 6
+ const w = o.w ?? 640
+ const cell = o.cell ?? Math.min(96, (w - left - 6) / o.cols)
+ const ch = o.cellH ?? cell
+ const h = top + ch * o.rows + 6
+ svg.replaceChildren()
+ svg.setAttribute('viewBox', `0 0 ${w} ${h}`)
+ // A small drawing scales up to the phone's width but not past a readable size on a desktop.
+ svg.style.maxWidth = `${Math.round(w * 1.3)}px`
+ const color = o.color ?? ((v) => heatTone(((v - (o.lo ?? 0)) / ((o.hi ?? 1) - (o.lo ?? 0) || 1))))
+ const x0 = left + (w - left - 6 - cell * o.cols) / 2
+ for (let c = 0; c < o.cols; c++) if (o.colLabels) svg.append(svgEl('text', { x: x0 + cell * (c + 0.5), y: 18, class: 'dk-tick', 'text-anchor': 'middle' }, o.colLabels[c]))
+ for (let r = 0; r < o.rows; r++) {
+ if (o.rowLabels) svg.append(svgEl('text', { x: x0 - 8, y: top + ch * (r + 0.5) + 4, class: 'dk-tick', 'text-anchor': 'end' }, o.rowLabels[r]))
+ for (let c = 0; c < o.cols; c++) {
+ const v = o.value(r, c)
+ const fill = color(v)
+ const m = o.mark?.(r, c) ?? ''
+ const g = svgEl('g', { class: `nn-cell ${m ? `nn-${m}` : ''}` })
+ const gap = ch < 20 ? 0.5 : 1.5
+ g.append(svgEl('rect', { x: x0 + cell * c + 1.5, y: top + ch * r + gap, width: cell - 3, height: ch - 2 * gap, rx: Math.min(10, ch / 6), fill }))
+ if (o.fmt !== null && ch >= 26) g.append(svgEl('text', { x: x0 + cell * (c + 0.5), y: top + ch * (r + 0.5) + 4, 'text-anchor': 'middle', class: 'nn-cell-text', fill: inkOn(fill), style: `font-size:${Math.max(9, Math.min(15, cell / 4.2))}px` }, (o.fmt ?? fmt)(v)))
+ if (o.onTap) {
+ g.setAttribute('role', 'button')
+ g.setAttribute('tabindex', '0')
+ g.addEventListener('click', () => o.onTap(r, c))
+ g.addEventListener('keydown', (ev) => { if (ev.key === 'Enter' || ev.key === ' ') { ev.preventDefault(); o.onTap(r, c) } })
+ }
+ svg.append(g)
+ }
+ }
+ return { cell, cellH: ch, x0, top, h }
+}
+
+/**
+ * A node-link diagram. layers: [[{ label, value, color }], ...] left to right; edges:
+ * [{ from: [layer, i], to: [layer, j], value, label }] coloured by tone(value, m). o: { w, h, m }.
+ */
+export function net(svg, layers, edges, o = {}) {
+ const w = o.w ?? 640, h = o.h ?? 300
+ svg.replaceChildren()
+ svg.setAttribute('viewBox', `0 0 ${w} ${h}`)
+ svg.style.maxWidth = `${Math.round(w * 1.3)}px`
+ const pos = layers.map((nodes, l) => nodes.map((_, i) => [70 + ((w - 140) * l) / Math.max(1, layers.length - 1), h / 2 + (i - (nodes.length - 1) / 2) * Math.min(86, (h - 60) / Math.max(1, nodes.length - 1 || 1))]))
+ for (const e of edges) {
+ const [a, b] = [pos[e.from[0]][e.from[1]], pos[e.to[0]][e.to[1]]]
+ const m = o.m ?? 1
+ svg.append(svgEl('line', { x1: a[0], y1: a[1], x2: b[0], y2: b[1], stroke: tone(e.value, m), 'stroke-width': 2 + 6 * Math.min(1, Math.abs(e.value) / m), 'stroke-linecap': 'round', opacity: 0.9 }))
+ if (e.label != null) {
+ const t = 0.62
+ svg.append(svgEl('text', { x: a[0] + (b[0] - a[0]) * t, y: a[1] + (b[1] - a[1]) * t - 6, class: 'nn-edge-label', 'text-anchor': 'middle' }, e.label))
+ }
+ }
+ layers.forEach((nodes, l) => nodes.forEach((n, i) => {
+ const [x, y] = pos[l][i]
+ svg.append(svgEl('circle', { cx: x, cy: y, r: 24, fill: n.color ?? '#0b1512', stroke: n.ring ?? COLORS[l % COLORS.length], 'stroke-width': 3 }))
+ svg.append(svgEl('text', { x, y: y + 5, 'text-anchor': 'middle', class: 'nn-node-text', fill: n.color ? inkOn(n.color) : '#fff' }, n.value ?? ''))
+ if (n.label) svg.append(svgEl('text', { x, y: y - 32, 'text-anchor': 'middle', class: 'dk-txt-dim' }, n.label))
+ }))
+}
+
+/** A heading question over the widget: the question the lesson opens with. */
+export const ask = (text) => el('p', { class: 'nn-ask' }, text)
diff --git a/apps/website/public/widgets/nn-lamb/card.png b/apps/website/public/widgets/nn-lamb/card.png
new file mode 100644
index 0000000000..148009ff1d
Binary files /dev/null and b/apps/website/public/widgets/nn-lamb/card.png differ
diff --git a/apps/website/public/widgets/nn-lamb/index.html b/apps/website/public/widgets/nn-lamb/index.html
new file mode 100644
index 0000000000..e815e8a364
--- /dev/null
+++ b/apps/website/public/widgets/nn-lamb/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+LAMB: one learning rate for layers of every size · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
LAMB: one learning rate for layers of every size
+
Scale a layer's weights and watch its LAMB step: it grows with the weight norm, so every layer moves by the same fraction of itself, until the norm passes 10 and the step stops growing.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-lamb/logic.js b/apps/website/public/widgets/nn-lamb/logic.js
new file mode 100644
index 0000000000..050f47f582
--- /dev/null
+++ b/apps/website/public/widgets/nn-lamb/logic.js
@@ -0,0 +1,48 @@
+// Generated by scripts/t27-logic.mjs from lamb.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 3dd28c18f0aa03e48a900bc0a78a02bf8a54812f695656dd4aa3c0f991682971
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-lamb/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABKgdgAn1/AX1gAX0Bf2AAAX1gAX8BfWAHf39/f39/fwBgAX0BfWACfX0BfQMLCgAAAQIDBAUDAwYEBQFwAQEBBQMBABEGCQF/AUGAgMAACwd8CAZtZW1vcnkCAAh0MjdfcG93aQAADXQyN19pc19maW5pdGUAAg90MjdfdmFsaWRfaW5wdXQAAw50MjdfdHdvX2xheWVycwAEDXQyN19zdGVwX25vcm0ABhB0MjdfZmlyc3RfdXBkYXRlAAcOdDI3X2luaXRfcHJvYmUACAqXDQoMACAAIAEQgYCAgAALKAEBfUMAAIA/IQICQANAIAFFDQEgAUF/aiEBIAAgApQhAgwACwsgAgsSACAAIABbIAAgAJMiACAAW3ELBwBDZmZmPwuSAwIGfwF9I4CAgIAAQYABayIBJICAgIAAIAFCAjcCRCABQQI2AjwgASABQRhqQQhqNgJMIAFBAjYCUCABIAFBKGpBCGo2AlQgAUICNwJYIAFCADcCKCABQgA3AjAgAUIANwIYIAFCADcCICABQoCAgIKEgIDAwAA3AhAgAUKAgID8g4CAgMAANwIIIAEgAUEoajYCQCABIAFBGGo2AjggAUKKro/hg4CAkMEANwJ4IAFC9/z++9P3jcM1NwJwIAFC76SM1OPMmbM/NwJoQQAhAiABQThqIQMCQANAIAJBEEYNASABIAFBCGogAmoiBEECIAJBgIDAgABqIAMgAUHoAGogAUHgAGoQhYCAgAAgASgCACEFIANBEGoiBiAGKAIAQQFqNgIAIAQgBCoCACAFKgIAkjgCACAEQQRqIgQgBCoCACAFKgIEkjgCACADQRRqIQMgAkEIaiECDAALCwJAAkAgAEEERw0AIAEoAlyzIQcMAQsgAUEIaiAAQQJ0aioCACEHCyABQYABaiSAgICAACAHC60DBQh9An8EfQN/AX1DAACAPyAFKgIIIgeTIQhDAACAPyAFKgIEIgmTIQogBSoCFCELIAUqAhAhDCAFKgIMIQ0gBSoCACEOIAQoAgghDyAEKAIAIRBDAACAPyAJIAQoAhBBAWoiBRCBgICAAJMhEUMAAIA/IAcgBRCBgICAAJMhEkMAAAAAIRNBACEFIAIhBEMAAAAAIRQCQANAIARFDQEgECAFaiIVIAkgFSoCAJQgCiADIAVqIhYqAgCUkjgCACAPIAVqIhcgByAXKgIAlCAWKgIAIhggCCAYlJSSIhg4AgAgBiAFaiAMIAEgBWoiFioCAJQgFSoCACARlSANIBggEpWRkpWSIhg4AgAgFCAYIBiUkiEUIAVBBGohBSAEQX9qIQQgEyAWKgIAIhggGJSSIRMMAAsLQwAAgD8hGAJAIBNDAAAAAF5FDQAgFEMAAAAAXkUNACATkSALEImAgIAAIBSRlSEYCyAYIA6MlCEYIAYhBSACIQQCQANAIARFDQEgBSAYIAUqAgCUOAIAIAVBBGohBSAEQX9qIQQMAAsLIAAgAjYCBCAAIAY2AgAL2gECAn8BfSOAgICAAEHgAGsiASSAgICAACABQoquj+GDgICQwQA3AlggAUL3/P770/eNwzU3AlAgAULvpIzU48yZsz83AkggAUICNwJAIAFBAjYCOCABQgA3AiQgAUIANwIcIAEgADgCFCABIAAgAJI4AhggASABQSRqNgI8IAEgAUEcajYCNCABQQhqIAFBFGpBAkGQgMCAACABQTRqIAFByABqIAFBLGoQhYCAgAAgASgCCCICKgIEIQAgAioCACEDIAFB4ABqJICAgIAAIAMgA5QgACAAlJKRC5kCAgJ/AX0jgICAgABB4ABrIgEkgICAgAAgAUEwakEAKAKggMCAADYCACABQQApApiAwIAANwMoQQAhAgJAA0AgAkEMRg0BIAFBHGogAmpBADYCACABQRBqIAJqQQA2AgAgAkEEaiECDAALCyABQgM3AkAgAUEDNgI4IAFCiq6P4YOAgJDBADcCWCABQvf8/vvT943DNTcCUCABQu+kjNTjzJmzPzcCSCABIAFBHGo2AjwgASABQRBqNgI0IAFBCGpBpIDAgABBA0GwgMCAACABQTRqIAFByABqIAFBKGoQhYCAgAACQAJAIABBA0cNACABKAIMsyEDDAELIAEoAgggAEECdGoqAgAhAwsgAUHgAGokgICAgAAgAwvDAQICfwF9I4CAgIAAQaAGayIBJICAgIAAAkBBkANFIgINACABQbyAwIAAQZAD/AoAAAsCQCACDQAgAUGQA2pBvIDAgABBkAP8CgAAC0EAIQICQANAIAJBkANGDQEgAUGQA2ogAmpBADYCACABIAJqQQA2AgAgAkEEaiECDAALC0MAAMhCIQMCQAJAAkACQCAADgQDAwABAgtDAAAAACEDDAILIAEqAgAhAwwBCyABKgKcBiEDCyABQaAGaiSAgICAACADC0UBAX8CQCAAIABcDQACQCABIAFbDQAgAA8LAkAgAbwgALwiAnNBf0oNACAAIAEgAkEASBsPCyAAIAEgACABXRshAQsgAQsL1gMBAEGAgMAAC8wDzczMPc3MTD6amZk+zczMPs3MzD3NzEw+AAAQQQAAEEEAABBBAACAPwAAAEAAAEBAzczMPc3MTD6amZk+AAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQ==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function initProbe(which) {
+ return W.t27_init_probe(int(which, 0, 255, "init_probe.which"));
+}
+function firstUpdate(index) {
+ return W.t27_first_update(int(index, 0, 4294967295, "first_update.index"));
+}
+function stepNorm(scale) {
+ return W.t27_step_norm(scale);
+}
+function twoLayers(which) {
+ return W.t27_two_layers(int(which, 0, 4294967295, "two_layers.which"));
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+function isFinite(x) {
+ return W.t27_is_finite(x) !== 0;
+}
+function powi(b, t) {
+ return W.t27_powi(b, int(t, 0, 4294967295, "powi.t"));
+}
+const __NOT_LOWERED__ = Object.freeze(["init_state", "compute_layer_update", "forward", "cfg", "step"]);
+export {
+ __NOT_LOWERED__,
+ firstUpdate,
+ initProbe,
+ isFinite,
+ powi,
+ stepNorm,
+ twoLayers,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-lamb/logic.t27 b/apps/website/public/widgets/nn-lamb/logic.t27
new file mode 100644
index 0000000000..5d9e8dfb42
--- /dev/null
+++ b/apps/website/public/widgets/nn-lamb/logic.t27
@@ -0,0 +1,246 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// LAMB parameter update with trust ratio clipping | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: built with `append` on `[]` and `range` / `len` / `pow`, so it never
+// compiled. The math was also not LAMB: v_hat used beta1's bias correction, the
+// "trust ratio" was ||g|| / ||v_hat|| and was computed, then never applied to the
+// step, and m and v were never written back. It is now You et al. (2020):
+// r = m_hat / (sqrt(v_hat) + eps) + wd * w, trust = min(||w||, clip) / ||r||,
+// w = w - lr * trust * r,
+// per layer, over buffers the caller owns.
+
+module Lamb;
+ use base::types;
+ use math::constants;
+ use ml::optimizer::adam;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_LR : f32 = 0.001;
+ const DEFAULT_BETA1 : f32 = 0.9;
+ const DEFAULT_BETA2 : f32 = 0.999;
+ const DEFAULT_EPSILON : f32 = 1e-6;
+ const DEFAULT_CLIP_THRESHOLD : f32 = 10.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const LAMBConfig = struct {
+ learning_rate : f32,
+ beta1 : f32,
+ beta2 : f32,
+ epsilon : f32,
+ weight_decay : f32,
+ clip_threshold : f32,
+ };
+
+ pub const LAMBState = struct {
+ m : []f32,
+ v : []f32,
+ step : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // m and v are the first num_params entries of the given buffers, zeroed; step 0
+ fn init_state(m: []f32, v: []f32, num_params: u32) -> LAMBState {
+ var i : usize = 0;
+ while (i < num_params) {
+ m[i] = 0.0;
+ v[i] = 0.0;
+ i = i + 1;
+ }
+ return LAMBState{ .m = m[0..num_params], .v = v[0..num_params], .step = 0 };
+ }
+
+ // powi (b^t for a whole t) comes from specs/ml/optimizer/adam.t27 through `use`
+
+ // Writes the moments for step (state.step + 1) into state.m / state.v and the update to
+ // ADD to params into `update`; returns update[0..n]
+ fn compute_layer_update(params: []const f32, grads: []const f32, state: LAMBState, config: LAMBConfig, update: []f32) -> []f32 {
+ const n = params.len;
+ const t = state.step + 1;
+ const c1 = 1.0 - powi(config.beta1, t);
+ const c2 = 1.0 - powi(config.beta2, t);
+ var w_norm : f32 = 0.0;
+ var r_norm : f32 = 0.0;
+ var i : usize = 0;
+ while (i < n) {
+ state.m[i] = config.beta1 * state.m[i] + (1.0 - config.beta1) * grads[i];
+ state.v[i] = config.beta2 * state.v[i] + (1.0 - config.beta2) * grads[i] * grads[i];
+ const r = (state.m[i] / c1) / (sqrt(state.v[i] / c2) + config.epsilon) + config.weight_decay * params[i];
+ update[i] = r;
+ w_norm = w_norm + params[i] * params[i];
+ r_norm = r_norm + r * r;
+ i = i + 1;
+ }
+ w_norm = sqrt(w_norm);
+ r_norm = sqrt(r_norm);
+ var trust : f32 = 1.0;
+ if (w_norm > 0.0 and r_norm > 0.0) {
+ trust = @min(w_norm, config.clip_threshold) / r_norm;
+ }
+ i = 0;
+ while (i < n) {
+ update[i] = -config.learning_rate * trust * update[i];
+ i = i + 1;
+ }
+ return update[0..n];
+ }
+
+ // One LAMB step over several layers packed back to back in `params` (layer k has
+ // sizes[k] entries), each with its own state; params change in place, each step + 1.
+ fn forward(params: []f32, grads: []const f32, sizes: []const usize, states: []LAMBState, config: LAMBConfig, scratch: []f32) -> void {
+ var start : usize = 0;
+ var k : usize = 0;
+ while (k < sizes.len) {
+ const end = start + sizes[k];
+ const upd = compute_layer_update(params[start..end], grads[start..end], states[k], config, scratch);
+ var j : usize = 0;
+ while (j < sizes[k]) {
+ params[start + j] = params[start + j] + upd[j];
+ j = j + 1;
+ }
+ states[k].step = states[k].step + 1;
+ start = end;
+ k = k + 1;
+ }
+ }
+
+ fn cfg(lr: f32) -> LAMBConfig {
+ return LAMBConfig{ .learning_rate = lr, .beta1 = DEFAULT_BETA1, .beta2 = DEFAULT_BETA2, .epsilon = DEFAULT_EPSILON, .weight_decay = 0.01, .clip_threshold = DEFAULT_CLIP_THRESHOLD };
+ }
+
+ // which: 0 = m.len, 1 = v.len, 2 = step, 3 = m[0], 4 = v[99]
+ fn init_probe(which: u8) -> f32 {
+ var m : [100]f32 = [9.0] ** 100;
+ var v : [100]f32 = [9.0] ** 100;
+ const st = init_state(&m, &v, 100);
+ if (which == 0) {
+ return @as(f32, @floatFromInt(st.m.len));
+ }
+ if (which == 1) {
+ return @as(f32, @floatFromInt(st.v.len));
+ }
+ if (which == 2) {
+ return @as(f32, @floatFromInt(st.step));
+ }
+ if (which == 3) {
+ return st.m[0];
+ }
+ return st.v[99];
+ }
+
+ // update[index] of the first step on params (1, 2, 3), grads (0.1, 0.2, 0.3); index 3 = its length
+ fn first_update(index: usize) -> f32 {
+ const params : [3]f32 = [1.0, 2.0, 3.0];
+ const grads : [3]f32 = [0.1, 0.2, 0.3];
+ var m : [3]f32 = undefined;
+ var v : [3]f32 = undefined;
+ var update : [3]f32 = [9.0] ** 3;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 3), cfg(0.001), &update);
+ if (index == 3) {
+ return @as(f32, @floatFromInt(u.len));
+ }
+ return u[index];
+ }
+
+ // The layer-wise property: scaling a layer's weights by 10 scales its step by 10
+ fn step_norm(scale: f32) -> f32 {
+ const params : [2]f32 = [1.0 * scale, 2.0 * scale];
+ const grads : [2]f32 = [0.1, 0.2];
+ var m : [2]f32 = undefined;
+ var v : [2]f32 = undefined;
+ var update : [2]f32 = undefined;
+ const u = compute_layer_update(¶ms, &grads, init_state(&m, &v, 2), cfg(0.001), &update);
+ return sqrt(u[0] * u[0] + u[1] * u[1]);
+ }
+
+ // Two layers (1, 2) and (3, 4), one step; which: 0..3 = new params, 4 = step of layer 1
+ fn two_layers(which: usize) -> f32 {
+ var params : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ const grads : [4]f32 = [0.1, 0.2, 0.3, 0.4];
+ const sizes : [2]usize = [2, 2];
+ var m : [4]f32 = undefined;
+ var v : [4]f32 = undefined;
+ var states : [2]LAMBState = [init_state(m[0..2], v[0..2], 2), init_state(m[2..4], v[2..4], 2)];
+ var scratch : [2]f32 = undefined;
+ forward(¶ms, &grads, &sizes, &states, cfg(0.001), &scratch);
+ if (which == 4) {
+ return @as(f32, @floatFromInt(states[1].step));
+ }
+ return params[which];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_BETA1;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test init_state_basic_case
+ given num_params = 100.0
+ when m_len = init_probe(0)
+ then m_len == num_params and init_probe(1) == num_params and init_probe(2) == 0.0 and init_probe(3) == 0.0 and init_probe(4) == 0.0
+
+ // Every update is negative (descent) and the step's norm is lr * min(||w||, 10) = 0.001 ||w||
+ test compute_layer_update_basic_case
+ given u0 = first_update(0)
+ when n = first_update(3)
+ then n == 3.0 and u0 < 0.0 and first_update(1) < 0.0 and first_update(2) < 0.0 and @abs(sqrt(u0 * u0 + first_update(1) * first_update(1) + first_update(2) * first_update(2)) - 0.001 * sqrt(@as(f32, 14.0))) < 0.0000001
+
+ // Each layer moves, and each layer's state advances
+ test forward_basic_case
+ given p0 = two_layers(0)
+ when p3 = two_layers(3)
+ then p0 != 1.0 and two_layers(1) != 2.0 and two_layers(2) != 3.0 and p3 != 4.0 and p0 < 1.0 and two_layers(4) == 1.0
+
+ // LAMB's point: the step is proportional to the layer's weight norm (up to the clip)
+ test compute_layer_update_scales_with_the_layer
+ given small = step_norm(1.0)
+ when large = step_norm(3.0)
+ then @abs(large / small - 3.0) < 0.001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // 0 < beta1 < 1
+ invariant lamb_constraint_0
+ assert valid_input() > 0.0 and valid_input() < 1.0
+
+ // 0 < beta2 < 1
+ invariant lamb_constraint_1
+ assert DEFAULT_BETA2 > 0.0 and DEFAULT_BETA2 < 1.0
+
+ // epsilon > 0
+ invariant lamb_constraint_2
+ assert DEFAULT_EPSILON > 0.0
+
+ // clip_threshold > 0
+ invariant lamb_constraint_3
+ assert DEFAULT_CLIP_THRESHOLD > 0.0
+
+ // name: trust_ratio -- capped by clip_threshold: a layer of norm 50 steps as one of norm 10
+ invariant lamb_constraint_4
+ assert DEFAULT_CLIP_THRESHOLD == 10.0
+
+ // name: lamb_update -- bias correction makes the first m_hat equal g
+ invariant lamb_constraint_5
+ assert powi(DEFAULT_BETA1, 1) == DEFAULT_BETA1 and DEFAULT_LR > 0.0
+
+ // "You et al. (2020) - Large Batch Optimization for Deep Learning: Training BERT in 76 minutes"
+ invariant lamb_constraint_6
+ assert powi(0.5, 3) == 0.125
+
+ // "Used in BERT, T5, and other large-scale Transformer training"
+ invariant lamb_constraint_7
+ assert DEFAULT_BETA2 > DEFAULT_BETA1
diff --git a/apps/website/public/widgets/nn-lamb/nn-lamb.t27 b/apps/website/public/widgets/nn-lamb/nn-lamb.t27
new file mode 100644
index 0000000000..4d550fb9e8
--- /dev/null
+++ b/apps/website/public/widgets/nn-lamb/nn-lamb.t27
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-lamb.t27 -- LAMB: a layer-wise trust ratio for large batches
+; Source of truth for public/widgets/nn-lamb/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-lamb/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/lamb.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 21 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the scale of the weights (1, 2) from 0.1 to 10. Every point is
+; step_norm(scale), the norm of the LAMB update for gradients (0.1, 0.2) at lr 0.001, and the tiles
+; two_layers, one step on two layers at once.
+
+; WHAT IT DOES NOT CLAIM: the clip at a norm of 10 is the spec's DEFAULT_CLIP_THRESHOLD; the flat
+; part is that cap, not a property of every LAMB.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_lamb;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-lamb";
+pub const TITLE : str = "LAMB: one learning rate for layers of every size";
+pub const DESCRIPTION : str = "Scale a layer's weights and watch its LAMB step: it grows with the weight norm, so every layer moves by the same fraction of itself, until the norm passes 10 and the step stops growing.";
+pub const IMAGE_ALT : str = "LAMB card at scale 2: a green line rises straight from 0 at scale 0 and bends flat at 0.010 once the scale passes about 4.5, with the cursor at 0.00447. Tiles read step norm 0.004472, ten times the weights 0.010000, and the spec's two layers after one step 0.9984 1.9984 and 2.9965 3.9964.";
+pub const HOOK : str = "Weights (2, 4) take a step of 0.00447 and weights (6, 12) one of 0.01: a step is 0.001 of the layer's norm, capped at 10. That is how LAMB trains large batches.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/lamb.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 21 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SCALE_LO_C : u8 = 10;
+pub const K_SCALE_HI_C : u16 = 1000;
+pub const K_Y_HI_U : u8 = 12;
+pub const K_Y_TICKS_U : [5]u8 = [0, 3, 6, 9, 12];
+pub const K_X_TICKS : [6]u8 = [0, 2, 4, 6, 8, 10];
+pub const K_PRESET_SCALE_C : [4]u16 = [200, 600, 10, 1000];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["scale 2", "scale 6, capped", "scale 0.1", "scale 10"];
+pub const SAY_ASK : str = "Can one learning rate suit a tiny layer and a huge one?";
+pub const SAY_SCALE : str = "weight scale";
+pub const SAY_NORM : str = "step norm";
+pub const SAY_PLOT_LABEL : str = "The norm of one LAMB step as the layer's weights are scaled";
+pub const SAY_NORM_X10 : str = "weights x 10";
+pub const SAY_LAYER_A : str = "layer 1 after a step";
+pub const SAY_LAYER_B : str = "layer 2 after a step";
+pub const SAY_NOTE : str = "LAMB computes an Adam-style update for each layer, then rescales it by the trust ratio min(||w||, 10) / ||update||. A big layer takes a big step and a small layer a small one.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/lamb.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_window_reaches_the_cap {
+ assert K_SCALE_HI_C == 1000;
+ assert K_PRESET_SCALE_C[1] > 448;
+ assert K_PRESET_SCALE_C[0] < 448;
+}
+
+test the_axis_holds_the_cap {
+ assert K_Y_HI_U > 10;
+ assert K_Y_TICKS_U[4] == K_Y_HI_U;
+ assert K_PRESET_SCALE_C[2] == K_SCALE_LO_C;
+}
diff --git a/apps/website/public/widgets/nn-lamb/tool.js b/apps/website/public/widgets/nn-lamb/tool.js
new file mode 100644
index 0000000000..da6ce788b0
--- /dev/null
+++ b/apps/website/public/widgets/nn-lamb/tool.js
@@ -0,0 +1,34 @@
+// nn-lamb/tool.js -- lesson 21 of neural-networks: one learning rate for layers of every size. LAMB
+// scales each layer's Adam-style update by a trust ratio, the layer's weight norm over its update
+// norm (the weight norm capped at 10), so the step is a fixed fraction of the layer. Every point is
+// step_norm and every tile two_layers of gHashTag/t27 specs/ml/optimizer/lamb.t27, compiled to wasm
+// (./logic.js): weights (1, 2) times a scale, gradients (0.1, 0.2), lr 0.001. Words are SAY_ and
+// numbers K_ constants of specs/widgets/nn-lamb.t27.
+import { stepNorm, twoLayers } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const S0 = centi(K('K_SCALE_LO_C', 10)), S1 = centi(K('K_SCALE_HI_C', 1000))
+const ctl = controls([
+ { key: 'k', label: say('SAY_SCALE'), min: S0, max: S1, step: 0.05, fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { k: K('K_PRESET_SCALE_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_NORM'), COLORS[0]]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 330, x0: S0, x1: S1, y0: 0, y1: K('K_Y_HI_U', 12) / 1000, xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS_U', []).map((t) => t / 1000), yfmt: (t) => fmt(t, 3), xlabel: say('SAY_SCALE'), left: 60 }
+ const p = plot(svg, o)
+ p.line(sample((k) => stepNorm(k), S0, S1, 240), COLORS[0], 3.5)
+ cursor(svg, p, s.k, [[stepNorm(s.k), COLORS[1]]])
+ dragX(svg, p, o, (k) => ctl.set('k', k))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_NORM'), value: fmt(stepNorm(s.k), 6), tone: 0 },
+ { label: say('SAY_NORM_X10'), value: fmt(stepNorm(s.k * 10), 6), tone: 1 },
+ { label: say('SAY_LAYER_A'), value: [0, 1].map((i) => fmt(twoLayers(i), 4)).join(' '), tone: 2 },
+ { label: say('SAY_LAYER_B'), value: [2, 3].map((i) => fmt(twoLayers(i), 4)).join(' '), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-lstm/card.png b/apps/website/public/widgets/nn-lstm/card.png
new file mode 100644
index 0000000000..160151aa47
Binary files /dev/null and b/apps/website/public/widgets/nn-lstm/card.png differ
diff --git a/apps/website/public/widgets/nn-lstm/index.html b/apps/website/public/widgets/nn-lstm/index.html
new file mode 100644
index 0000000000..172d0cee1b
--- /dev/null
+++ b/apps/website/public/widgets/nn-lstm/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+LSTM gates: how a cell decides what to keep · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
LSTM gates: how a cell decides what to keep
+
Set the biases of the forget, input and output gates and the old memory: the new memory is forget x old + input x candidate, and the output shows tanh of it through the output gate.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-lstm/logic.js b/apps/website/public/widgets/nn-lstm/logic.js
new file mode 100644
index 0000000000..92a14e93f3
--- /dev/null
+++ b/apps/website/public/widgets/nn-lstm/logic.js
@@ -0,0 +1,48 @@
+// Generated by scripts/t27-logic.mjs from lstm_single.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: ef4ffa30e7adf81c8467768ec637ba33643c7872f6746cf244cdaa12eff68385
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-lstm/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABWQxgAAF/YAZ9fX19fX8BfWAFf39/f38AYAd/f39/f39/AX1gAX0BfWABfwF9YAJ/fQF9YAF/AGAAAX1gCH9/f39/f39/AGALfX9/f39/f39/f38AYAJ9fwF9AxIRAAECAwQEBQYHCAkKBQYEBAsEBQFwAQEBBQMBABEGCQF/AUGAgMAACwd2CAZtZW1vcnkCAA90MjdfdmFsaWRfaW5wdXQAAA10MjdfYmlhc19vbmx5AAEJdDI3X3Nsb3BlAAYXdDI3X3plcm9fdXBzdHJlYW1fdG90YWwACQh0MjdfZ3JhZAAMCHQyN19sb3NzAA0LdDI3X3NpZ21vaWQADgqaMhEEAEEDC5IEAQF/I4CAgIAAQfACayIGJICAgIAAAkBBwAFFDQAgBkEIakEAQcAB/AsACyAGIAM4AsABIAYgAjgCkAEgBiABOAJgIAYgADgCMCAGQgA3AsgBIAZBADYC1AEgBiAEOALQASAGQoCAwIiEgICIwQA3A9gBIAZCgIDAiISAgIjBADcD4AEgBkECNgL0ASAGQQI2AuwBIAZBAjYC1AIgBiAGQcABajYC0AIgBkEENgLMAiAGIAZBsAFqNgLIAiAGQQY2AsQCIAYgBkGYAWo2AsACIAZBAjYCvAIgBiAGQZABajYCuAIgBkEENgK0AiAGIAZBgAFqNgKwAiAGQQY2AqwCIAYgBkHoAGo2AqgCIAZBAjYCpAIgBiAGQeAAajYCoAIgBkEENgKcAiAGIAZB0ABqNgKYAiAGQQY2ApQCIAYgBkE4ajYCkAIgBkECNgKMAiAGIAZBMGo2AogCIAZBBDYChAIgBiAGQSBqNgKAAiAGQQY2AvwBIAYgBkHQAWo2AvABIAYgBkHIAWo2AugBIAYgBkEIajYC+AEgBkECNgLkAiAGQQI2AtwCIAZCg4CAgCA3A+gCIAYgBkHgAWo2AuACIAYgBkHYAWo2AtgCQYCAwIAAIAZB6AFqIAZB+AFqIAZB2AJqIAZB6AJqEIKAgIAAIAZB4AFqIAZB2AFqIAUbKgIAIQMgBkHwAmokgICAgAAgAwvsAgIQfwR9IAQoAgQhBSADKAIIIQYgAygCACEHIAIoAlghCCACKAJQIQkgAigCSCEKIAIoAkAhCyACKAI4IQwgAigCMCENIAIoAighDiACKAIgIQ8gAigCGCEQIAIoAhAhESACKAIIIRIgAigCACETIAEoAgghFCABKAIAIQNBACECAkADQCAFIAJGDQEgEyASIBEgACADIAIgBBCDgICAAIwQj4CAgAAhFSAQIA8gDiAAIAMgAiAEEIOAgIAAIRYgDSAMIAsgACADIAIgBBCDgICAABCEgICAACEXIBaMEI+AgIAAIRYgCiAJIAggACADIAIgBBCDgICAACEYIAYgF0MAAIA/IBVDAACAP5KVlEMAAIA/IBZDAACAP5KVIBQqAgCUkiIVOAIAIAdDAACAPyAYjBCPgICAAEMAAIA/kpUgFRCEgICAAJQ4AgAgB0EEaiEHIBRBBGohFCAGQQRqIQYgAkEBaiECDAALCwufAQIBfwF9IAAgBigCACIHIAVsQQJ0aiEAIAIgBUECdGoqAgAhCCAGKAIEIQYDfQJAIAcNACABIAYgBWxBAnRqIQcCQANAIAZFDQEgBkF/aiEGIAggByoCACAEKgIAlJIhCCAEQQRqIQQgB0EEaiEHDAALCyAIDwsgB0F/aiEHIAggACoCACADKgIAlJIhCCADQQRqIQMgAEEEaiEADAALC+wBAwF/AX0BfyOAgICAAEEQayIBJICAgIAAAkACQCAAiyICvCIDQdW+svgDSQ0AAkAgA0GAgICJBE0NAEMAAAAAIACVQwAAgD+SIQIMAgtDAACAP0MAAABAIAIgApIQhYCAgABDAAAAQJKVkyECDAELAkAgA0H4iov0A0sNAAJAIANBgICABEkNACACQwAAAMCUEIWAgIAAIgKMIAJDAAAAQJKVIQIMAgsgASAAIACUOAIMIAFBDGohAwwBCyACIAKSEIWAgIAAIgIgAkMAAABAkpUhAgsgAUEQaiSAgICAACACjCACIAC8QQBIGwu4BQQBfwF9An8DfSOAgICAAEEQayIBJICAgIAAAkACQCAAIABbDQBDAADAfyECDAELQwAAgL8hAiAAQwAAgP9bDQACQAJAAkACQAJAAkACQAJAIAC8IgNB/////wdxIgRBw/DWjARLDQAgBEGY5MX1A00NASAEQZKrlPwDTw0CQX8hBCADQX9MDQMgAEOAcTG/kiECQ9H3FzchBUEBIQQMBgsgBEGAgID8B0sNBCADQQBIDQcgAEOAcbFCXkUNASAAQwAAAH+UIQIMBwsgBEGAgICYA0kNAkMAAAAAIQVBACEEDAULIABDO6q4P5RDAAAAPyAAmJL8ACIEsiICQ9H3FzeUIQUgACACQ4BxMb+UkiECDAMLIABDgHExP5IhAkPR9xe3IQUMAgsgBEH///8DSw0AIAEgACAAlDgCDCABQQxqIQQLIAAhAgwCCyACIAIgBZMiAJMgBZMhBQsgACAAQwAAAD+UIgaUIgIgAiACQxAwzzqUQ2iICL2SlEMAAIA/kiIHQwAAQEAgBiAHlJMiBpNDAADAQCAAIAaUk5WUIQYCQCAEDQAgACAAIAaUIAKTkyECDAELIAAgBiAFk5QgBZMgApMhAgJAAkACQAJAIARBAWoOAwACAQILIAAgApNDAAAAP5RDAAAAv5IhAgwDCyAAQwAAgL5dDQEgACACkyIAIACSQwAAgD+SIQIMAgsgBEEXdCIDQYCAgPwDar4hBQJAIARBOEsNAEGAgID8AyADa74hBgJAIARBFksNAEMAAIA/IAaTIAAgApOSIAWUIQIMAwsgACACIAaSk0MAAIA/kiAFlCECDAILIAAgApNDAACAP5IiACAAkkMAAAB/lCAAIAWUIARBgAFGG0MAAIC/kiECDAELIAIgAEMAAAA/kpNDAAAAwJQhAgsgAUEQaiSAgICAACACCyMAIABDbxKDOhCHgICAACAAQ28Sg7oQh4CAgACTQ28SAzuVC74FAgJ/A30jgICAgABBgANrIgIkgICAgAAgAkEMahCIgICAACACQYCAgIIENgLUASACQoCAgPyDgICAwAA3AswBIAJCzZmz7tOZs6Y+NwLYASACQpqz5vTTmbPmPjcC4AECQCAAQX9qQQNPDQAgAkHMAWogAEECdGpBfGoiAyABIAMqAgCSOAIACwJAAkACQAJAIABBfnFBfGoOAwACAQILIAJB2AFqIABBAnRqQXBqIgAgASAAKgIAkjgCAAwCCyACQeABaiAAQQJ0akFoaiIAIAEgACoCAJI4AgAMAQsgAEEIRw0AIAIgASACKgJokjgCaAsgAkIANwPwASACQgA3A+gBIAJBAjYChAIgAkECNgL8ASACQQI2AuQCIAJBBDYC3AIgAkEGNgLUAiACQQI2AswCIAJBBDYCxAIgAkEGNgK8AiACQQI2ArQCIAJBBDYCrAIgAkEGNgKkAiACQQI2ApwCIAJBBDYClAIgAkEGNgKMAiACIAJBxAFqNgLgAiACIAJBtAFqNgLYAiACIAJBnAFqNgLQAiACIAJBlAFqNgLIAiACIAJBhAFqNgLAAiACIAJB7ABqNgK4AiACIAJB5ABqNgKwAiACIAJB1ABqNgKoAiACIAJBPGo2AqACIAIgAkE0ajYCmAIgAiACQSRqNgKQAiACIAJB4AFqNgKAAiACIAJB2AFqNgL4ASACIAJBDGo2AogCIAJBAjYC9AIgAkECNgLsAiACQoOAgIAgNwP4AiACIAJB8AFqNgLwAiACIAJB6AFqNgLoAiACQcwBaiACQfgBaiACQYgCaiACQegCaiACQfgCahCCgICAACACKgLsASEBIAIqAugBIQQgAioC8AEhBSACKgL0ASEGIAJBgANqJICAgIAAIARDAAAAP5QgAUOamZm+lJIgBUPNzEw+lJIgBkPNzMw9lJILWgECf0EAIQEgACECAkADQCABQTBGDQEgAiABQQdsQQNqQf//A3FBC3CzQwAAIEGVQwAAAL+SOAIAIAJBBGohAiABQQFqIQEMAAsLIABCgICA/IOAgMA/NwJYC6oHAgN/AX0jgICAgABBoAVrIgAkgICAgAAgAEEIahCIgICAAEEAIQEgAEHgAWoiAkEANgIAIABCmrPm9NOZs+Y+NwLQASAAQs2Zs+7TmbOmPjcCyAEgAEIANwPYASAAQgA3A+gBIABCADcD8AECQEHAAUUNACAAQfgBakEAQcAB/AsACyAAQQI2AsQDIABBAjYCvAMgAEECNgKkBCAAIABBCGpBuAFqNgKgBCAAQQQ2ApwEIAAgAEEIakGoAWo2ApgEIABBBjYClAQgACAAQQhqQZABajYCkAQgAEECNgKMBCAAIABBCGpBiAFqNgKIBCAAQQQ2AoQEIAAgAEEIakH4AGo2AoAEIABBBjYC/AMgACAAQQhqQeAAajYC+AMgAEECNgL0AyAAIABBCGpB2ABqNgLwAyAAQQQ2AuwDIAAgAEEIakHIAGo2AugDIABBBjYC5AMgACAAQQhqQTBqNgLgAyAAQQI2AtwDIAAgAEEIakEoajYC2AMgAEEENgLUAyAAIABBCGpBGGo2AtADIABBBjYCzAMgACAAQdABajYCwAMgACAAQcgBajYCuAMgACAAQQhqNgLIAyAAQQI2ArwEIABCg4CAgCA3A6gEIABBAjYCtAQgAEEGNgLEBCAAIABB+AFqQRhqNgLIBCAAQQQ2AswEIAAgAEH4AWpBKGo2AtAEIABBAjYC1AQgACAAQfgBakEwajYC2AQgAEEGNgLcBCAAIABB+AFqQcgAajYC4AQgAEEENgLkBCAAIABB+AFqQdgAajYC6AQgAEECNgLsBCAAIABB+AFqQeAAajYC8AQgAEEGNgL0BCAAIABB+AFqQfgAajYC+AQgAEEENgL8BCAAIABB+AFqQYgBajYCgAUgAEECNgKEBSAAIABB+AFqQZABajYCiAUgAEEGNgKMBSAAIABB+AFqQagBajYCkAUgAEEENgKUBSAAIABB+AFqQbgBajYCmAUgAEECNgKcBSAAIABB8AFqNgK4BCAAIABB6AFqNgKwBCAAIABB+AFqNgLABEGMgMCAAEGMgMCAACAAQbgDaiAAQcgDaiAAQagEaiAAQdgBaiAAQbAEaiAAQcAEahCKgICAACAAKgLYAYsgACoC3AGLkiACKgIAi5IgACoC6AGLkiAAKgLsAYuSIAAqAvABi5IgACoC9AGLkiEDAkADQCABQcABRg0BIAMgAEH4AWogAWoqAgCLkiEDIAFBBGohAQwACwsgAEGgBWokgICAgAAgAwuBBgIbfwh9IAcoAlghCCAHKAJQIQkgBygCSCEKIAcoAkAhCyAHKAI4IQwgBygCMCENIAcoAighDiAHKAIgIQ8gBygCGCEQIAcoAhAhESAHKAIIIRIgBygCACETIAYoAgghFCAGKAIAIQYgBCgCBCEVIAQoAgAhByADKAJYIRYgAygCUCEXIAMoAkghGCADKAJAIRkgAygCOCEaIAMoAjAhGyADKAIoIRwgAygCICEdIAMoAhghHiADKAIQIR8gAygCCCEgIAMoAgAhISACKAIIISIgAigCACEDIAUhAgNAAkAgBw0AIAYhAiAVIQcDQAJAIAcNAEEAIQcCQANAIBUgB0YNASAhICAgH0GAgMCAACADIAcgBBCDgICAAIwQj4CAgAAhIyAeIB0gHEGAgMCAACADIAcgBBCDgICAACEkIBsgGiAZQYCAwIAAIAMgByAEEIOAgIAAEISAgIAAISUgJIwQj4CAgAAhJCAYIBcgFkGAgMCAACADIAcgBBCDgICAACEmICVDAACAPyAjQwAAgD+SlSIjlEMAAIA/ICRDAACAP5KVIiQgIioCAJSSEISAgIAAIScgJowQj4CAgAAhJiAiKgIAISggFCAkIAEqAgAgACoCACIpQwAAgD8gJkMAAIA/kpUiKpRDAACAPyAnICeUk5SSIiaUOAIAQwAAgD8gI5MgIyAlICaUlJQgISAgIBMgEiARIAMgByAEIAUgBhCLgICAAEMAAIA/ICSTICQgKCAmlJSUIB4gHSAQIA8gDiADIAcgBCAFIAYQi4CAgABDAACAPyAlICWUkyAjICaUlCAbIBogDSAMIAsgAyAHIAQgBSAGEIuAgIAAQwAAgD8gKpMgKiAnICmUlJQgGCAXIAogCSAIIAMgByAEIAUgBhCLgICAACAUQQRqIRQgAUEEaiEBIABBBGohACAiQQRqISIgB0EBaiEHDAALCw8LIAJBADYCACACQQRqIQIgB0F/aiEHDAALCyACQQA2AgAgAkEEaiECIAdBf2ohBwwACwuAAgEDfyADIAgoAgAiCyAHbEECdCIMaiENIAEgDGohDCAIKAIEIQNBACEIA0ACQCALDQAgBCADIAdsQQJ0IgtqIQggAiALaiELAkADQCADRQ0BIAggCCoCACAAIAYqAgCUkjgCACAKIAoqAgAgACALKgIAlJI4AgAgCkEEaiEKIAZBBGohBiAIQQRqIQggC0EEaiELIANBf2ohAwwACwsgBSAHQQJ0aiIIIAAgCCoCAJI4AgAPCyANIAhqIgEgASoCACAAIAhBgIDAgABqKgIAlJI4AgAgCSAIaiIBIAEqAgAgACAMIAhqKgIAlJI4AgAgCEEEaiEIIAtBf2ohCwwACwu0BwIBfwF9I4CAgIAAQaAFayIBJICAgIAAIAFBCGoQiICAgAAgAUHgAWpBADYCACABQpqz5vTTmbPmPjcC0AEgAULNmbPu05mzpj43AsgBIAFCADcD2AEgAUIANwPoASABQgA3A/ABAkBBwAFFDQAgAUH4AWpBAEHAAfwLAAsgAUECNgLEAyABQQI2ArwDIAFBAjYCpAQgASABQQhqQbgBajYCoAQgAUEENgKcBCABIAFBCGpBqAFqNgKYBCABQQY2ApQEIAEgAUEIakGQAWo2ApAEIAFBAjYCjAQgASABQQhqQYgBajYCiAQgAUEENgKEBCABIAFBCGpB+ABqNgKABCABQQY2AvwDIAEgAUEIakHgAGo2AvgDIAFBAjYC9AMgASABQQhqQdgAajYC8AMgAUEENgLsAyABIAFBCGpByABqNgLoAyABQQY2AuQDIAEgAUEIakEwajYC4AMgAUECNgLcAyABIAFBCGpBKGo2AtgDIAFBBDYC1AMgASABQQhqQRhqNgLQAyABQQY2AswDIAEgAUHQAWo2AsADIAEgAUHIAWo2ArgDIAEgAUEIajYCyAMgAUKDgICAIDcDqAQgAUECNgK0BCABQQI2ArwEIAFBBjYCxAQgASABQfgBakEYajYCyAQgAUEENgLMBCABIAFB+AFqQShqNgLQBCABQQI2AtQEIAEgAUH4AWpBMGo2AtgEIAFBBjYC3AQgASABQfgBakHIAGo2AuAEIAFBBDYC5AQgASABQfgBakHYAGo2AugEIAFBAjYC7AQgASABQfgBakHgAGo2AvAEIAFBBjYC9AQgASABQfgBakH4AGo2AvgEIAFBBDYC/AQgASABQfgBakGIAWo2AoAFIAFBAjYChAUgASABQfgBakGQAWo2AogFIAFBBjYCjAUgASABQfgBakGoAWo2ApAFIAFBBDYClAUgASABQfgBakG4AWo2ApgFIAFBAjYCnAUgASABQegBajYCsAQgASABQfABajYCuAQgASABQfgBajYCwARBlIDAgABBnIDAgAAgAUG4A2ogAUHIA2ogAUGoBGogAUHYAWogAUGwBGogAUHABGoQioCAgAACQAJAIABBf2pBAksNACABQdgBaiAAQQJ0akF8aioCACECDAELAkACQAJAIABBfnFBfGoOAwACAQILIAFB6AFqIABBAnRqQXBqKgIAIQIMAgsgAUHwAWogAEECdGpBaGoqAgAhAgwBCyABKgLUAiECCyABQaAFaiSAgICAACACCwwAIAAgARCHgICAAAsXAEMAAIA/IACMEI+AgIAAQwAAgD+SlQvUAgIDfwN9AkACQCAAIABcDQAgALwiAUEfdiECAkACQAJAAkACQAJAAkAgAUH/////B3EiA0HP2LqVBEsNACADQZjkxfUDTQ0BIANBkquU/ANNDQIMBAsgA0GAgID8B0sNBgJAIAFBAEgNACADQZjkxZUESQ0AIABDAAAAf5QPCyABQX9KDQNDAAAAACEEIANBtOO/lgRLDQcMAwsgA0GAgIDIA00NAUEAIQNDAAAAACEFIAAhBAwECyACQQFzIAJrIQMMAgsgAEMAAIA/kg8LIABDO6q4P5QgAkECdCoCpIDAgACS/AAhAwsgACADsiIEQwByMb+UkiIAIARDjr6/NZQiBZMhBAsgACAEIAQgBCAElCIGIAZDFVI1u5RDj6oqPpKUkyIGlEMAAABAIAaTlSAFk5JDAACAP5IhBCADRQ0BIAQgAxCQgICAAA8LIAAhBAsgBAumAgEEfwJAAkAgACAAXA0AIAC8IgJB/////wdxQYCAgPwHTw0AIAJBgICAgHhxIQNBCCACQQF0IgRnayAEQRh2IARBgICACEkbIQQCQCABQQBIDQACQCABQf4BIARrTA0AQwAAgH8gAJgPCwJAIAQgAWoiBUEASg0AIAIgAXQgA3K+DwsCQCAEQQBKDQAgAkEBIARrdEH///8DcSAFQRd0ciADcr4PCyABQRd0IAJqvg8LIAFBACAEa0oNAQJAIAFBaSAEa04NACADvg8LQYCAgARBACAEQQBKGyACQf///wNxckEAIAFrIgNBASABIARqayIBIAMgAUgbQX9qIgF2QQFqQQF2IAJoIAFBH3FGQX9zcb4gAJghAAsgAA8LIAFBF3QgAmq+Cws1AQBBgIDAAAssAACAPwAAAEAAAEBAAAAAAAAAAAAAAAA/mpmZvs3MTD7NzMw9AAAAPwAAAL8=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function sigmoid(z) {
+ return W.t27_sigmoid(z);
+}
+function loss(which, delta) {
+ return W.t27_loss(int(which, 0, 4294967295, "loss.which"), delta);
+}
+function grad(which) {
+ return W.t27_grad(int(which, 0, 4294967295, "grad.which"));
+}
+function zeroUpstreamTotal() {
+ return W.t27_zero_upstream_total();
+}
+function slope(which) {
+ return W.t27_slope(int(which, 0, 4294967295, "slope.which"));
+}
+function biasOnly(bI, bF, bG, bO, cPrev, which) {
+ return W.t27_bias_only(bI, bF, bG, bO, cPrev, int(which, 0, 255, "bias_only.which"));
+}
+function validInput() {
+ return W.t27_valid_input() >>> 0;
+}
+const __NOT_LOWERED__ = Object.freeze(["pre", "forward", "spread", "backward", "cfg", "weights_of", "fill"]);
+export {
+ __NOT_LOWERED__,
+ biasOnly,
+ grad,
+ loss,
+ sigmoid,
+ slope,
+ validInput,
+ zeroUpstreamTotal
+};
diff --git a/apps/website/public/widgets/nn-lstm/logic.t27 b/apps/website/public/widgets/nn-lstm/logic.t27
new file mode 100644
index 0000000000..bc16437835
--- /dev/null
+++ b/apps/website/public/widgets/nn-lstm/logic.t27
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// LSTM solves vanishing gradient problem | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: called `@sigmoid` and `@tanh`, which Zig does not have; each gate weight
+// was one number per INPUT, indexed `i % hidden_size`, so it was not an LSTM; and
+// every test asserted `true`. Now each gate has its own matrices, W_x
+// (hidden x input) and W_h (hidden x hidden), and a bias:
+// i = s(z_i), f = s(z_f), g = tanh(z_g), o = s(z_o), z = W_x x + W_h h + b
+// c' = f c + i g, h' = o tanh(c')
+// backward gives every gradient of one step, and the tests check it against finite
+// differences.
+
+module LstmCell;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const FORGET_BIAS_INIT : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const LSTMConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ };
+
+ pub const LSTMState = struct {
+ h : []f32,
+ c : []f32,
+ };
+
+ pub const LSTMWeights = struct {
+ W_ii : []f32,
+ W_hi : []f32,
+ b_i : []f32,
+ W_if : []f32,
+ W_hf : []f32,
+ b_f : []f32,
+ W_ig : []f32,
+ W_hg : []f32,
+ b_g : []f32,
+ W_io : []f32,
+ W_ho : []f32,
+ b_o : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Pre-activation of one gate for hidden unit u: W_x[u] . x + W_h[u] . h + b[u]
+ fn pre(w_x: []const f32, w_h: []const f32, b: []const f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig) -> f32 {
+ var z : f32 = b[u];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + w_x[u * config.input_size + j] * x[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + w_h[u * config.hidden_size + k] * h[k];
+ k = k + 1;
+ }
+ return z;
+ }
+
+ fn forward(input: []const f32, state: LSTMState, weights: LSTMWeights, output_state: LSTMState, config: LSTMConfig) -> void {
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ output_state.c[u] = fg * state.c[u] + ig * gg;
+ output_state.h[u] = og * tanh(output_state.c[u]);
+ u = u + 1;
+ }
+ }
+
+ // Adds dz * (x, h, 1) into one gate's weight gradients, and dz * W into grad_input / grad_state.h
+ fn spread(dz: f32, w_x: []const f32, w_h: []const f32, gw_x: []f32, gw_h: []f32, gb: []f32, x: []const f32, h: []const f32, u: usize, config: LSTMConfig, grad_input: []f32, grad_h: []f32) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ gw_x[u * config.input_size + j] = gw_x[u * config.input_size + j] + dz * x[j];
+ grad_input[j] = grad_input[j] + dz * w_x[u * config.input_size + j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ gw_h[u * config.hidden_size + k] = gw_h[u * config.hidden_size + k] + dz * h[k];
+ grad_h[k] = grad_h[k] + dz * w_h[u * config.hidden_size + k];
+ k = k + 1;
+ }
+ gb[u] = gb[u] + dz;
+ }
+
+ // One step back. grad_output = dL/dh', grad_cell = dL/dc' from later steps. Writes
+ // dL/dx, dL/dh, dL/dc and ADDS the weight gradients into grad_weights.
+ // do = dh' tanh(c'); dc = dh' o (1 - tanh^2 c') + grad_cell
+ // dz_o = do o (1 - o), dz_f = dc c f (1 - f), dz_i = dc g i (1 - i), dz_g = dc i (1 - g^2)
+ // dL/dc = dc f
+ fn backward(grad_output: []const f32, grad_cell: []const f32, input: []const f32, state: LSTMState, weights: LSTMWeights, config: LSTMConfig, grad_input: []f32, grad_state: LSTMState, grad_weights: LSTMWeights) -> void {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ grad_input[j] = 0.0;
+ j = j + 1;
+ }
+ var u : usize = 0;
+ while (u < config.hidden_size) {
+ grad_state.h[u] = 0.0;
+ u = u + 1;
+ }
+ u = 0;
+ while (u < config.hidden_size) {
+ const ig = sigmoid(pre(weights.W_ii, weights.W_hi, weights.b_i, input, state.h, u, config));
+ const fg = sigmoid(pre(weights.W_if, weights.W_hf, weights.b_f, input, state.h, u, config));
+ const gg = tanh(pre(weights.W_ig, weights.W_hg, weights.b_g, input, state.h, u, config));
+ const og = sigmoid(pre(weights.W_io, weights.W_ho, weights.b_o, input, state.h, u, config));
+ const c_new = fg * state.c[u] + ig * gg;
+ const tc = tanh(c_new);
+ const dc = grad_output[u] * og * (1.0 - tc * tc) + grad_cell[u];
+ const dz_o = grad_output[u] * tc * og * (1.0 - og);
+ const dz_f = dc * state.c[u] * fg * (1.0 - fg);
+ const dz_i = dc * gg * ig * (1.0 - ig);
+ const dz_g = dc * ig * (1.0 - gg * gg);
+ grad_state.c[u] = dc * fg;
+ spread(dz_i, weights.W_ii, weights.W_hi, grad_weights.W_ii, grad_weights.W_hi, grad_weights.b_i, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_f, weights.W_if, weights.W_hf, grad_weights.W_if, grad_weights.W_hf, grad_weights.b_f, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_g, weights.W_ig, weights.W_hg, grad_weights.W_ig, grad_weights.W_hg, grad_weights.b_g, input, state.h, u, config, grad_input, grad_state.h);
+ spread(dz_o, weights.W_io, weights.W_ho, grad_weights.W_io, grad_weights.W_ho, grad_weights.b_o, input, state.h, u, config, grad_input, grad_state.h);
+ u = u + 1;
+ }
+ }
+
+ // The fixture: input 3, hidden 2. All 48 weights live in one buffer, per gate
+ // W_x (6), W_h (4), b (2), in the gate order i, f, g, o.
+ fn cfg() -> LSTMConfig {
+ return LSTMConfig{ .input_size = 3, .hidden_size = 2 };
+ }
+
+ fn weights_of(w: []f32) -> LSTMWeights {
+ return LSTMWeights{
+ .W_ii = w[0..6], .W_hi = w[6..10], .b_i = w[10..12],
+ .W_if = w[12..18], .W_hf = w[18..22], .b_f = w[22..24],
+ .W_ig = w[24..30], .W_hg = w[30..34], .b_g = w[34..36],
+ .W_io = w[36..42], .W_ho = w[42..46], .b_o = w[46..48],
+ };
+ }
+
+ // Deterministic weights in [-0.5, 0.5]; the forget biases start at FORGET_BIAS_INIT
+ fn fill(w: []f32) -> void {
+ var k : usize = 0;
+ while (k < 48) {
+ w[k] = @as(f32, @floatFromInt((k * 7 + 3) % 11)) / 10.0 - 0.5;
+ k = k + 1;
+ }
+ w[22] = FORGET_BIAS_INIT;
+ w[23] = FORGET_BIAS_INIT;
+ }
+
+ // L = 0.5 h'_0 - 0.3 h'_1 + 0.2 c'_0 + 0.1 c'_1 after one step, with one input, one h or
+ // one c entry moved by `delta` (which: 0 = none, 1 + j = x_j, 4 + k = h_k, 6 + k = c_k,
+ // 8 = b_f[1])
+ fn loss(which: usize, delta: f32) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ var x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ if (which >= 1 and which <= 3) {
+ x[which - 1] = x[which - 1] + delta;
+ }
+ if (which >= 4 and which <= 5) {
+ h[which - 4] = h[which - 4] + delta;
+ }
+ if (which >= 6 and which <= 7) {
+ c[which - 6] = c[which - 6] + delta;
+ }
+ if (which == 8) {
+ w[23] = w[23] + delta;
+ }
+ var h_out : [2]f32 = [0.0] ** 2;
+ var c_out : [2]f32 = [0.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ return 0.5 * h_out[0] - 0.3 * h_out[1] + 0.2 * c_out[0] + 0.1 * c_out[1];
+ }
+
+ // The analytic gradient of that loss, same `which` numbering
+ fn grad(which: usize) -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const grad_output : [2]f32 = [0.5, -0.3];
+ const grad_cell : [2]f32 = [0.2, 0.1];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&grad_output, &grad_cell, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ if (which >= 1 and which <= 3) {
+ return gx[which - 1];
+ }
+ if (which >= 4 and which <= 5) {
+ return gh[which - 4];
+ }
+ if (which >= 6 and which <= 7) {
+ return gc[which - 6];
+ }
+ return gw[23];
+ }
+
+ // Sum of |every gradient| when grad_output and grad_cell are both 0
+ fn zero_upstream_total() -> f32 {
+ var w : [48]f32 = undefined;
+ fill(&w);
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.1, 0.2];
+ var c : [2]f32 = [0.3, 0.4];
+ const zeros : [2]f32 = [0.0, 0.0];
+ var gx : [3]f32 = [0.0] ** 3;
+ var gh : [2]f32 = [0.0] ** 2;
+ var gc : [2]f32 = [0.0] ** 2;
+ var gw : [48]f32 = [0.0] ** 48;
+ backward(&zeros, &zeros, &x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), cfg(), &gx, LSTMState{ .h = &gh, .c = &gc }, weights_of(&gw));
+ var total : f32 = @abs(gx[0]) + @abs(gx[1]) + @abs(gx[2]) + @abs(gh[0]) + @abs(gh[1]) + @abs(gc[0]) + @abs(gc[1]);
+ var k : usize = 0;
+ while (k < 48) {
+ total = total + @abs(gw[k]);
+ k = k + 1;
+ }
+ return total;
+ }
+
+ fn slope(which: usize) -> f32 {
+ return (loss(which, 0.001) - loss(which, -0.001)) / 0.002;
+ }
+
+ // which: 0 = h'_0, 1 = c'_0 for a 1-unit cell with every weight 0 and the given biases
+ fn bias_only(b_i: f32, b_f: f32, b_g: f32, b_o: f32, c_prev: f32, which: u8) -> f32 {
+ var w : [48]f32 = [0.0] ** 48;
+ w[10] = b_i;
+ w[22] = b_f;
+ w[34] = b_g;
+ w[46] = b_o;
+ const x : [3]f32 = [1.0, 2.0, 3.0];
+ var h : [2]f32 = [0.0, 0.0];
+ var c : [2]f32 = [c_prev, 0.0];
+ var h_out : [2]f32 = [9.0] ** 2;
+ var c_out : [2]f32 = [9.0] ** 2;
+ forward(&x, LSTMState{ .h = &h, .c = &c }, weights_of(&w), LSTMState{ .h = &h_out, .c = &c_out }, cfg());
+ if (which == 0) {
+ return h_out[0];
+ }
+ return c_out[0];
+ }
+
+ fn valid_input() -> usize {
+ return cfg().input_size;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // All weights 0: every gate is s(0) = 0.5 or tanh(0) = 0, so c' = 0.5 c and h' = 0.5 tanh(c')
+ test forward_basic_case
+ given c_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 1)
+ when h_new = bias_only(0.0, 0.0, 0.0, 0.0, 0.8, 0)
+ then @abs(c_new - 0.4) < 0.000001 and @abs(h_new - 0.5 * tanh(c_new)) < 0.000001 and @abs(h_new - 0.1899745) < 0.000001
+
+ // A closed forget gate (b_f = -30) erases the old cell; an open one (b_f = 30) keeps it
+ test forward_basic_lstm
+ given erased = bias_only(0.0, -30.0, 0.0, 0.0, 0.8, 1)
+ when kept = bias_only(-30.0, 30.0, 0.0, 0.0, 0.8, 1)
+ then @abs(erased) < 0.000001 and @abs(kept - 0.8) < 0.000001
+
+ // A closed input gate and forget gate leave c' = 0, so h' = 0 whatever the output gate does
+ test forward_with_empty_input
+ given h_new = bias_only(-30.0, -30.0, 1.0, 30.0, 0.8, 0)
+ then @abs(h_new) < 0.000001
+
+ // dL/dx_j against central differences
+ test backward_basic_case
+ given g0 = grad(1)
+ when g2 = grad(3)
+ then @abs(g0 - slope(1)) < 0.0005 and @abs(grad(2) - slope(2)) < 0.0005 and @abs(g2 - slope(3)) < 0.0005
+
+ // dL/dh_k, dL/dc_k and dL/db_f[1] against central differences
+ test backward_basic_lstm
+ given gh0 = grad(4)
+ when gc1 = grad(7)
+ then @abs(gh0 - slope(4)) < 0.0005 and @abs(grad(5) - slope(5)) < 0.0005 and @abs(grad(6) - slope(6)) < 0.0005 and @abs(gc1 - slope(7)) < 0.0005 and @abs(grad(8) - slope(8)) < 0.0005
+
+ // With no upstream gradient nothing flows back
+ test backward_with_empty_grad
+ given total = zero_upstream_total()
+ then total == 0.0 and @abs(grad(1)) > 0.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.size() == config.input_size
+ invariant lstm_cell_constraint_0
+ assert valid_input() == 3
+
+ // state.h.size() == config.hidden_size
+ invariant lstm_cell_constraint_1
+ assert cfg().hidden_size == 2
+
+ // state.c.size() == config.hidden_size; the forget bias starts open
+ invariant lstm_cell_constraint_2
+ assert FORGET_BIAS_INIT > 0.0
diff --git a/apps/website/public/widgets/nn-lstm/nn-lstm.t27 b/apps/website/public/widgets/nn-lstm/nn-lstm.t27
new file mode 100644
index 0000000000..aeaa04a2ba
--- /dev/null
+++ b/apps/website/public/widgets/nn-lstm/nn-lstm.t27
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-lstm.t27 -- an LSTM cell's gates: forget, input, output
+; Source of truth for public/widgets/nn-lstm/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-lstm/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/recurrent/lstm_single.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 23 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets four gate biases from -6 to 6 and the old memory from -2 to 2. With
+; every weight 0, each gate is sigmoid(bias); the new memory and output are bias_only(b_i, b_f, b_g,
+; b_o, c_prev, which) of the spec.
+
+; WHAT IT DOES NOT CLAIM: one step of a 1-unit cell with all weights 0, so the gates see only their
+; biases; that is the spec's own fixture.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_lstm;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-lstm";
+pub const TITLE : str = "LSTM gates: how a cell decides what to keep";
+pub const DESCRIPTION : str = "Set the biases of the forget, input and output gates and the old memory: the new memory is forget x old + input x candidate, and the output shows tanh of it through the output gate.";
+pub const IMAGE_ALT : str = "LSTM card, remember: gate bars forget 0.998, input 0.002 and output 0.998; memory bars old 1.000, new 0.998 and output 0.759. Tiles read new memory 0.9975, output 0.7587, forget 0.9975 and input 0.0025.";
+pub const HOOK : str = "With the forget gate open (0.9975) and the input gate shut, a memory of 1 comes out as 0.9975: an LSTM can hold a value for a long time.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/recurrent/lstm_single.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 23 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_BIAS_HI_C : u16 = 600;
+pub const K_C_LO_C : i16 = -200;
+pub const K_C_HI_C : u16 = 200;
+pub const K_CELL_LO : i8 = -3;
+pub const K_CELL_HI : u8 = 3;
+pub const K_GATE_TICKS_C : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_CELL_TICKS : [5]i8 = [-2, -1, 0, 1, 2];
+pub const K_PRESET_BF_C : [4]i16 = [600, -600, -600, 600];
+pub const K_PRESET_BI_C : [4]i16 = [-600, 0, 600, -600];
+pub const K_PRESET_BG_C : [4]i16 = [0, 0, 600, 0];
+pub const K_PRESET_BO_C : [4]i16 = [600, 600, 600, -600];
+pub const K_PRESET_C_C : [4]i16 = [100, 100, 100, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Remember", "Forget", "Write", "Hide"];
+pub const SAY_ASK : str = "How does a memory cell decide what to keep and what to forget?";
+pub const SAY_BF : str = "forget bias";
+pub const SAY_BI : str = "input bias";
+pub const SAY_BG : str = "candidate bias";
+pub const SAY_BO : str = "output bias";
+pub const SAY_C_PREV : str = "old memory c";
+pub const SAY_GATES_LABEL : str = "The three gates, each a sigmoid of its bias";
+pub const SAY_CELL_LABEL : str = "The memory before and after, and the output";
+pub const SAY_FORGET : str = "forget";
+pub const SAY_INPUT : str = "input";
+pub const SAY_OUTPUT : str = "output";
+pub const SAY_C_OLD : str = "old c";
+pub const SAY_C_NEW : str = "new c";
+pub const SAY_H_NEW : str = "output h";
+pub const SAY_NOTE : str = "c' = f c + i g and h' = o tanh(c'). When f is near 1 and i near 0, c passes through almost untouched, and so does its gradient: the long memory the plain cell lacked.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/recurrent/lstm_single.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_biases_are_symmetric {
+ assert K_BIAS_HI_C == 600;
+ assert K_PRESET_BF_C[1] + K_BIAS_HI_C == 0;
+ assert K_PRESET_BG_C[2] == K_BIAS_HI_C;
+}
+
+test the_presets_cover_the_gates {
+ assert K_PRESET_BF_C[0] > 0;
+ assert K_PRESET_BI_C[0] < 0;
+ assert K_PRESET_BO_C[3] < 0;
+ assert K_PRESET_C_C[0] <= K_C_HI_C;
+}
diff --git a/apps/website/public/widgets/nn-lstm/tool.js b/apps/website/public/widgets/nn-lstm/tool.js
new file mode 100644
index 0000000000..396a38c432
--- /dev/null
+++ b/apps/website/public/widgets/nn-lstm/tool.js
@@ -0,0 +1,35 @@
+// nn-lstm/tool.js -- lesson 23 of neural-networks: how a cell decides what to keep. An LSTM has three
+// gates, each a sigmoid: forget scales the old memory, input lets new content in, output decides
+// how much of the memory to show. Here every weight is 0, so each gate is the sigmoid of its bias.
+// The gates are sigmoid and the new memory and output are bias_only of gHashTag/t27
+// specs/ml/recurrent/lstm_single.t27, compiled to wasm (./logic.js). Words are SAY_ and numbers K_
+// constants of specs/widgets/nn-lstm.t27.
+import { biasOnly, sigmoid } from './logic.js'
+import { COLORS, K, W, ask, bars, canvas, centi, controls, el, fmt, root, say, source, tiles } from '../nn-kit.js'
+
+const B = centi(K('K_BIAS_HI_C', 600))
+const bias = (key, label) => ({ key, label, min: -B, max: B, step: 0.5, fmt: (v) => fmt(v, 1) })
+const ctl = controls([
+ bias('bf', say('SAY_BF')), bias('bi', say('SAY_BI')), bias('bg', say('SAY_BG')), bias('bo', say('SAY_BO')),
+ { key: 'c', label: say('SAY_C_PREV'), min: centi(K('K_C_LO_C', -200)), max: centi(K('K_C_HI_C', 200)), step: 0.1, fmt: (v) => fmt(v, 1) },
+], { names: W.SAY_PRESET_NAMES, values: { bf: K('K_PRESET_BF_C', []).map(centi), bi: K('K_PRESET_BI_C', []).map(centi), bg: K('K_PRESET_BG_C', []).map(centi), bo: K('K_PRESET_BO_C', []).map(centi), c: K('K_PRESET_C_C', []).map(centi) } }, draw)
+
+const gates = canvas('nn-bars', say('SAY_GATES_LABEL'))
+const cell = canvas('nn-bars', say('SAY_CELL_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('div', { class: 'nn-two' }, el('div', {}, el('p', { class: 'dk-label' }, say('SAY_GATES_LABEL')), gates), el('div', {}, el('p', { class: 'dk-label' }, say('SAY_CELL_LABEL')), cell)), stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const g = [sigmoid(s.bf), sigmoid(s.bi), sigmoid(s.bo)]
+ bars(gates, { values: g, labels: [say('SAY_FORGET'), say('SAY_INPUT'), say('SAY_OUTPUT')], colors: [COLORS[4], COLORS[2], COLORS[5]], lo: 0, hi: 1, h: 240, ticks: K('K_GATE_TICKS_C', []).map(centi), fmt: (v) => fmt(v, 3) })
+ const c2 = biasOnly(s.bi, s.bf, s.bg, s.bo, s.c, 1), h2 = biasOnly(s.bi, s.bf, s.bg, s.bo, s.c, 0)
+ bars(cell, { values: [s.c, c2, h2], labels: [say('SAY_C_OLD'), say('SAY_C_NEW'), say('SAY_H_NEW')], colors: [COLORS[6], COLORS[0], COLORS[1]], lo: K('K_CELL_LO', -3), hi: K('K_CELL_HI', 3), h: 240, ticks: K('K_CELL_TICKS', []), fmt: (v) => fmt(v, 3) })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_C_NEW'), value: fmt(c2, 4), tone: 0 },
+ { label: say('SAY_H_NEW'), value: fmt(h2, 4), tone: 1 },
+ { label: say('SAY_FORGET'), value: fmt(g[0], 4), tone: 4 },
+ { label: say('SAY_INPUT'), value: fmt(g[1], 4), tone: 2 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-maxpool/card.png b/apps/website/public/widgets/nn-maxpool/card.png
new file mode 100644
index 0000000000..055a66da37
Binary files /dev/null and b/apps/website/public/widgets/nn-maxpool/card.png differ
diff --git a/apps/website/public/widgets/nn-maxpool/index.html b/apps/website/public/widgets/nn-maxpool/index.html
new file mode 100644
index 0000000000..7583979a4e
--- /dev/null
+++ b/apps/website/public/widgets/nn-maxpool/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Max pooling: what a network keeps when it shrinks a picture · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Max pooling: what a network keeps when it shrinks a picture
+
Pick a window size and stride and tap an output: max pooling keeps the largest value of its window and remembers where it was, and the gradient later goes to that one cell.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-maxpool/logic.js b/apps/website/public/widgets/nn-maxpool/logic.js
new file mode 100644
index 0000000000..be364622d8
--- /dev/null
+++ b/apps/website/public/widgets/nn-maxpool/logic.js
@@ -0,0 +1,32 @@
+// Generated by scripts/t27-logic.mjs from maxpool2d_layer.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 12627dff26da63deb67347730d28228fda9b60ca09779e37d052f4a94f3e7e0a
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-maxpool/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABGgRgAX8BfWADf39/AGAEf39/fwF9YAJ/fwF/AwUEAAECAwQFAXABAQEFAwEAEQYJAX8BQYCAwAALBzkEBm1lbW9yeQIAD3QyN19iYWNrd2FyZF9hdAAAC3QyN19wb29sX2F0AAIMdDI3X291dF9zaWRlAAMKnAYEjQICA38BfSOAgICAAEGAAWsiASSAgICAACABQSBqQgA3AwAgAUIANwMYIAFCgICAgBA3AjQgAUKCgICAIDcCLCABQQE6ADwgAUEBOgBAIAFBLGogAUEIaiABQRhqEIGAgIAAAkBBwABFDQAgAUHAAGpBwIDAgABBwAD8CgAAC0EAIQIDfQJAIAJBwABHDQBBACECAkADQCACQRBGDQEgAUHAAGogAUEYaiACaigCAEECdGoiAyADKgIAIAJBgIHAgABqKgIAkjgCACACQQRqIQIMAAsLIAFBwABqIABBAnRqKgIAIQQgAUGAAWokgICAgAAgBA8LIAFBwABqIAJqQQA2AgAgAkEEaiECDAALC8oCARR/IAAoAgQiA0EEdCEEIANBAnQhBUEEIAAoAgAiBmsgA24iB0EBaiEIQQAhCUGAgMCAACEKQQAhCwJAA0AgCyAHSw0BIAsgCGwhDCALIAVsIQ1BACEOIAohDyAJIRADQAJAAkAgDiAHSw0AIA4gA2wgDWohEUEAIRIgDyETIBAhFANAIBIgBkYNAiATIRUgFCEWIAYhAAJAA0AgAEUNASAWIBEgFSoCACARQQJ0QYCAwIAAaioCAF4bIREgFUEEaiEVIBZBAWohFiAAQX9qIQAMAAsLIBNBEGohEyAUQQRqIRQgEkEBaiESDAALCyAKIARqIQogCSAFaiEJIAtBAWohCwwCCyABIA4gDGpBAnQiAGogEUECdEGAgMCAAGoqAgA4AgAgAiAAaiARNgIAIA8gBWohDyAQIANqIRAgDkEBaiEODAALCwsLsAECAn8BfSOAgICAAEHgAGsiBCSAgICAAAJAQSRFIgUNACAEQQBBJPwLAAsCQCAFDQAgBEEkakEAQST8CwALIARCgICAgBA3AlAgBCABNgJMIAQgADYCSCAEQQE6AFggBEEBOgBcIARByABqIAQgBEEkahCBgICAAAJAAkAgAg0AIAQgA0ECdGoqAgAhBgwBCyAEQSRqIANBAnRqKAIAsyEGCyAEQeAAaiSAgICAACAGCw0AQQQgAGsgAW5BAWoLC5oBAQBBgIDAAAuQAQAAgD8AAABAAACgQAAAwEAAAEBAAACAQAAAAEEAAOBAAAAAAAAAgL8AAEDAAAAAwAAAgMAAABBBAADAwAAAoMAAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAACAPwAAAEAAAEBAAACAQA==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function outSide(kernel, stride) {
+ return W.t27_out_side(int(kernel, 0, 4294967295, "out_side.kernel"), int(stride, 0, 4294967295, "out_side.stride")) >>> 0;
+}
+function poolAt(kernel, stride, which, index) {
+ return W.t27_pool_at(int(kernel, 0, 4294967295, "pool_at.kernel"), int(stride, 0, 4294967295, "pool_at.stride"), int(which, 0, 255, "pool_at.which"), int(index, 0, 4294967295, "pool_at.index"));
+}
+function backwardAt(index) {
+ return W.t27_backward_at(int(index, 0, 4294967295, "backward_at.index"));
+}
+const __NOT_LOWERED__ = Object.freeze(["out_size", "forward", "backward", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ backwardAt,
+ outSide,
+ poolAt
+};
diff --git a/apps/website/public/widgets/nn-maxpool/logic.t27 b/apps/website/public/widgets/nn-maxpool/logic.t27
new file mode 100644
index 0000000000..7b972a41a5
--- /dev/null
+++ b/apps/website/public/widgets/nn-maxpool/logic.t27
@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Reduce spatial dimensions | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward and backward compared two values and threw them away
+// (placeholders), and every test asserted `result != undefined`. Max pooling now
+// runs over one H x W channel (row-major) with a k x k window and a stride. It
+// records each window's argmax, and backward sends each gradient back to that
+// element only. Padding and dilation are carried in the config but not read: the
+// layer assumes padding 0 and dilation 1.
+
+module Maxpool2d;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const MaxPool2DConfig = struct {
+ kernel_size : u32,
+ stride : u32,
+ padding : u32,
+ dilation : u32,
+ return_indices : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // Output side for an input side n: (n - k) / stride + 1
+ fn out_size(n: u32, config: MaxPool2DConfig) -> u32 {
+ return (n - config.kernel_size) / config.stride + 1;
+ }
+
+ // output[oy][ox] = max of the k x k window at (oy * stride, ox * stride);
+ // indices[oy][ox] = the flat input index that held it
+ fn forward(input: []const f32, height: u32, width: u32, config: MaxPool2DConfig, output: []f32, indices: []u32) -> void {
+ const oh = out_size(height, config);
+ const ow = out_size(width, config);
+ var oy : u32 = 0;
+ while (oy < oh) {
+ var ox : u32 = 0;
+ while (ox < ow) {
+ var best_index : u32 = (oy * config.stride) * width + ox * config.stride;
+ var ky : u32 = 0;
+ while (ky < config.kernel_size) {
+ var kx : u32 = 0;
+ while (kx < config.kernel_size) {
+ const idx : u32 = (oy * config.stride + ky) * width + ox * config.stride + kx;
+ if (input[idx] > input[best_index]) {
+ best_index = idx;
+ }
+ kx = kx + 1;
+ }
+ ky = ky + 1;
+ }
+ output[oy * ow + ox] = input[best_index];
+ indices[oy * ow + ox] = best_index;
+ ox = ox + 1;
+ }
+ oy = oy + 1;
+ }
+ }
+
+ // grad_input[indices[o]] += grad_output[o]; every other input gets 0
+ fn backward(grad_output: []const f32, indices: []const u32, grad_input: []f32) -> void {
+ var i : usize = 0;
+ while (i < grad_input.len) {
+ grad_input[i] = 0.0;
+ i = i + 1;
+ }
+ var o : usize = 0;
+ while (o < grad_output.len) {
+ grad_input[indices[o]] = grad_input[indices[o]] + grad_output[o];
+ o = o + 1;
+ }
+ }
+
+ fn cfg(kernel: u32, stride: u32) -> MaxPool2DConfig {
+ return MaxPool2DConfig{ .kernel_size = kernel, .stride = stride, .padding = 0, .dilation = 1, .return_indices = true };
+ }
+
+ // The output side on the 4 x 4 fixture for a k x k window and a stride.
+ // out_size takes a MaxPool2DConfig struct and does not lower to WASM; this
+ // takes two integers, so a host reads the side from the compiled spec.
+ fn out_side(kernel: u32, stride: u32) -> u32 {
+ return out_size(4, cfg(kernel, stride));
+ }
+
+ // The 4 x 4 fixture
+ // 1 2 5 6
+ // 3 4 8 7
+ // 0 -1 -3 -2
+ // -4 9 -6 -5
+ fn pool_at(kernel: u32, stride: u32, which: u8, index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [9]f32 = [0.0] ** 9;
+ var indices : [9]u32 = [0] ** 9;
+ forward(&input, 4, 4, cfg(kernel, stride), &output, &indices);
+ if (which == 0) {
+ return output[index];
+ }
+ return @as(f32, @floatFromInt(indices[index]));
+ }
+
+ // 2 x 2, stride 2, gradients (1, 2, 3, 4) for the four windows; grad_input[index]
+ fn backward_at(index: usize) -> f32 {
+ const input : [16]f32 = [1.0, 2.0, 5.0, 6.0, 3.0, 4.0, 8.0, 7.0, 0.0, -1.0, -3.0, -2.0, -4.0, 9.0, -6.0, -5.0];
+ var output : [4]f32 = [0.0] ** 4;
+ var indices : [4]u32 = [0] ** 4;
+ forward(&input, 4, 4, cfg(2, 2), &output, &indices);
+ const grad_output : [4]f32 = [1.0, 2.0, 3.0, 4.0];
+ var grad_input : [16]f32 = [9.0] ** 16;
+ backward(&grad_output, &indices, &grad_input);
+ return grad_input[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // 2 x 2, stride 2: windows' maxima are 4, 8, 9, -2
+ test forward_basic_case
+ given first = pool_at(2, 2, 0, 0)
+ when last = pool_at(2, 2, 0, 3)
+ then first == 4.0 and pool_at(2, 2, 0, 1) == 8.0 and pool_at(2, 2, 0, 2) == 9.0 and last == -2.0
+
+ // The argmax indices: 4 at 5, 8 at 6, 9 at 13, -2 at 11
+ test forward_max_finding
+ given first = pool_at(2, 2, 1, 0)
+ when third = pool_at(2, 2, 1, 2)
+ then first == 5.0 and pool_at(2, 2, 1, 1) == 6.0 and third == 13.0 and pool_at(2, 2, 1, 3) == 11.0
+
+ // 3 x 3, stride 1 gives a 2 x 2 output whose windows overlap
+ test forward_overlapping_windows
+ given first = pool_at(3, 1, 0, 0)
+ when last = pool_at(3, 1, 0, 3)
+ then first == 8.0 and pool_at(3, 1, 0, 1) == 8.0 and pool_at(3, 1, 0, 2) == 9.0 and last == 9.0
+
+ test backward_basic_case
+ given at_max = backward_at(5)
+ when elsewhere = backward_at(0)
+ then at_max == 1.0 and elsewhere == 0.0
+
+ // Each window's gradient lands on its argmax only: the sum is conserved
+ test backward_gradient_propagation
+ given total = backward_at(5) + backward_at(6) + backward_at(13) + backward_at(11)
+ when others = backward_at(0) + backward_at(1) + backward_at(4) + backward_at(15)
+ then total == 10.0 and others == 0.0 and backward_at(13) == 3.0
+
+ // The four window/stride pairs the nn-maxpool widget offers, on the 4 x 4 fixture:
+ // 2 x 2 stride 2 -> 2, 3 x 3 stride 1 -> 2, 2 x 2 stride 1 -> 3, 4 x 4 stride 1 -> 1
+ test out_side_of_each_widget_config
+ given halved = out_side(2, 2)
+ when whole = out_side(4, 1)
+ then halved == 2 and out_side(3, 1) == 2 and out_side(2, 1) == 3 and whole == 1
+
+ // Edges: a 1 x 1 window at stride 1 keeps the side; a stride that does not
+ // divide (4 - k) rounds down (3 x 3 at stride 2 fits one window, 1 x 1 at
+ // stride 3 fits two)
+ test out_side_identity_and_floor
+ given same = out_side(1, 1)
+ when floored = out_side(3, 2)
+ then same == 4 and floored == 1 and out_side(1, 3) == 2
+
+ invariant maxpool_output_size_formula
+ assert out_size(4, cfg(2, 2)) == 2 and out_size(4, cfg(3, 1)) == 2 and out_size(5, cfg(2, 2)) == 2
diff --git a/apps/website/public/widgets/nn-maxpool/nn-maxpool.t27 b/apps/website/public/widgets/nn-maxpool/nn-maxpool.t27
new file mode 100644
index 0000000000..0cfa3a452c
--- /dev/null
+++ b/apps/website/public/widgets/nn-maxpool/nn-maxpool.t27
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-maxpool.t27 -- max pooling: keep the largest value of each window, and where it was
+; Source of truth for public/widgets/nn-maxpool/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-maxpool/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/maxpool2d_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 11 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader picks one of four window and stride pairs and taps an output. Outputs and
+; argmax indices are pool_at(kernel, stride, 0 or 1, i) and the output side is out_side(kernel,
+; stride); for 2 x 2 stride 2 the gradient view is backward_at(i), the spec's output gradients
+; 1, 2, 3, 4.
+
+; WHAT IT DOES NOT CLAIM: the grid is the spec's fixture, repeated as K_INPUT; the outputs and
+; argmax come from the compiled fn, so a K_INPUT that drifted would show a max that is not in its
+; window.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_maxpool;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-maxpool";
+pub const TITLE : str = "Max pooling: what a network keeps when it shrinks a picture";
+pub const DESCRIPTION : str = "Pick a window size and stride and tap an output: max pooling keeps the largest value of its window and remembers where it was, and the gradient later goes to that one cell.";
+pub const IMAGE_ALT : str = "Max-pool card, 2 x 2 windows with stride 2: the spec's 4 x 4 grid in pink and green, with cell 13 (value 9) outlined inside its window; the 2 x 2 output reads 4, 8, 9, -2 with 9 outlined. Tiles read max 9, argmax 13, output 2 x 2 and gradient at the max 3.";
+pub const HOOK : str = "On the spec's 4 x 4 grid, 2 x 2 windows with stride 2 keep 4, 8, 9 and -2; their gradients land on cells 5, 6, 13 and 11 and nowhere else.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/maxpool2d_layer.t27 (t27 master 2e53089d5): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 11 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SIDE : u8 = 4;
+pub const K_INPUT : [16]i8 = [1, 2, 5, 6, 3, 4, 8, 7, 0, -1, -3, -2, -4, 9, -6, -5];
+pub const K_KERNELS : [4]u8 = [2, 3, 2, 4];
+pub const K_STRIDES : [4]u8 = [2, 1, 1, 1];
+pub const K_PRESET_CFG : [3]u8 = [0, 1, 2];
+pub const K_PRESET_CELL : [3]u8 = [2, 0, 4];
+pub const K_PRESET_GRAD : [3]u8 = [0, 0, 0];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["2 x 2, stride 2", "3 x 3, stride 1", "2 x 2, stride 1"];
+pub const SAY_ASK : str = "What does a network keep when it shrinks a picture?";
+pub const SAY_CONFIG : str = "window, stride";
+pub const SAY_CONFIG_NAMES : [4]str = ["2 x 2 / 2", "3 x 3 / 1", "2 x 2 / 1", "4 x 4 / 1"];
+pub const SAY_SHOW_GRAD : str = "show the gradient (2 x 2 / 2)";
+pub const SAY_IN_LABEL : str = "The spec's 4 x 4 input, the max of the window outlined";
+pub const SAY_GRAD_LABEL : str = "The gradient each input receives";
+pub const SAY_OUT_LABEL : str = "The output: tap a cell";
+pub const SAY_MAX : str = "max";
+pub const SAY_ARGMAX : str = "from cell";
+pub const SAY_OUT_SIDE : str = "output";
+pub const SAY_GRAD_AT_MAX : str = "gradient there";
+pub const SAY_NO_GRAD : str = "2 x 2 / 2 only";
+pub const SAY_NOTE : str = "A max pool forgets every value but one per window, which makes the network care less about exactly where a feature sits. In backpropagation the whole gradient of a window goes to its maximum.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/maxpool2d_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_four_configs_are_the_spec_window_stride_pairs {
+ assert K_KERNELS[0] == 2;
+ assert K_STRIDES[0] == 2;
+ assert K_KERNELS[1] == 3;
+ assert K_STRIDES[1] == 1;
+ assert K_KERNELS[2] == 2;
+ assert K_STRIDES[2] == 1;
+ assert K_KERNELS[3] == K_SIDE;
+ assert K_STRIDES[3] == 1;
+}
+
+test the_grid_is_the_spec_fixture {
+ assert K_INPUT[5] == 4;
+ assert K_INPUT[6] == 8;
+ assert K_INPUT[13] == 9;
+ assert K_INPUT[11] + 2 == 0;
+}
diff --git a/apps/website/public/widgets/nn-maxpool/tool.js b/apps/website/public/widgets/nn-maxpool/tool.js
new file mode 100644
index 0000000000..ee50f11cbb
--- /dev/null
+++ b/apps/website/public/widgets/nn-maxpool/tool.js
@@ -0,0 +1,45 @@
+// nn-maxpool/tool.js -- lesson 11 of neural-networks: what a network keeps when it shrinks a
+// picture. Max pooling keeps the largest value of each window and remembers where it was; learning
+// then sends each window's gradient to that one cell. Every output and argmax is pool_at, the
+// output side out_side, every gradient backward_at of gHashTag/t27
+// specs/ml/layers/maxpool2d_layer.t27, compiled to wasm (./logic.js), on the spec's 4 x 4 fixture
+// (K_INPUT, held equal to it by the tests below the spec).
+// Words are SAY_ and numbers K_ constants of specs/widgets/nn-maxpool.t27.
+import { backwardAt, outSide, poolAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fmt, heat, root, say, source, tiles, tone } from '../nn-kit.js'
+
+const SIDE = K('K_SIDE', 4)
+const INPUT = K('K_INPUT', [])
+const KS = K('K_KERNELS', []), ST = K('K_STRIDES', [])
+const ctl = controls([
+ { key: 'cfg', label: say('SAY_CONFIG'), kind: 'choice', options: W.SAY_CONFIG_NAMES ?? [] },
+ { key: 'cell', kind: 'hidden', min: 0, max: 8 },
+ { key: 'grad', label: say('SAY_SHOW_GRAD'), kind: 'toggle' },
+], { names: W.SAY_PRESET_NAMES, values: { cfg: K('K_PRESET_CFG', []), cell: K('K_PRESET_CELL', []), grad: K('K_PRESET_GRAD', []) } }, draw)
+
+const img = canvas('nn-heat', say('SAY_IN_LABEL'))
+const out = canvas('nn-heat', say('SAY_OUT_LABEL'))
+const stat = el('div')
+const inLabel = el('p', { class: 'dk-label' })
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('div', { class: 'nn-two' }, el('div', {}, inLabel, img), el('div', {}, el('p', { class: 'dk-label' }, say('SAY_OUT_LABEL')), out)), stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const k = KS[s.cfg], st = ST[s.cfg], n = outSide(k, st)
+ const cell = Math.min(s.cell, n * n - 1)
+ const oy = Math.floor(cell / n), ox = cell % n
+ const inWin = (r, c) => r >= oy * st && r < oy * st + k && c >= ox * st && c < ox * st + k
+ const arg = poolAt(k, st, 1, cell)
+ // The gradient path exists for the spec's 2 x 2, stride 2 case only (backward_at).
+ const grad = !!s.grad && s.cfg === 0
+ inLabel.textContent = grad ? say('SAY_GRAD_LABEL') : say('SAY_IN_LABEL')
+ heat(img, { rows: SIDE, cols: SIDE, value: (r, c) => (grad ? backwardAt(r * SIDE + c) : INPUT[r * SIDE + c]), color: (v) => tone(v, 9), w: 400, cell: 92, fmt: (v) => fmt(v, 0), mark: (r, c) => (r * SIDE + c === arg ? 'on' : inWin(r, c) ? '' : 'dim') })
+ heat(out, { rows: n, cols: n, value: (r, c) => poolAt(k, st, 0, r * n + c), color: (v) => tone(v, 9), w: 400, cell: Math.min(150, 300 / n), fmt: (v) => fmt(v, 0), mark: (r, c) => (r * n + c === cell ? 'on' : ''), onTap: (r, c) => ctl.set('cell', r * n + c) })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_MAX'), value: fmt(poolAt(k, st, 0, cell), 0), tone: 0 },
+ { label: say('SAY_ARGMAX'), value: fmt(arg, 0), tone: 1 },
+ { label: say('SAY_OUT_SIDE'), value: `${n} x ${n}`, tone: 2 },
+ { label: say('SAY_GRAD_AT_MAX'), value: s.cfg === 0 ? fmt(backwardAt(arg), 0) : say('SAY_NO_GRAD'), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-mse/card.png b/apps/website/public/widgets/nn-mse/card.png
new file mode 100644
index 0000000000..433a46d3dd
Binary files /dev/null and b/apps/website/public/widgets/nn-mse/card.png differ
diff --git a/apps/website/public/widgets/nn-mse/index.html b/apps/website/public/widgets/nn-mse/index.html
new file mode 100644
index 0000000000..b08b5150f9
--- /dev/null
+++ b/apps/website/public/widgets/nn-mse/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Mean squared error: how wrong a guess is, as one number · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Mean squared error: how wrong a guess is, as one number
+
Move three predictions against the targets 1, 2 and 3: the loss is the mean of the squared misses, its bowl is a parabola, and each gradient points its prediction back at the target.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-mse/logic.js b/apps/website/public/widgets/nn-mse/logic.js
new file mode 100644
index 0000000000..7a297bf67b
--- /dev/null
+++ b/apps/website/public/widgets/nn-mse/logic.js
@@ -0,0 +1,32 @@
+// Generated by scripts/t27-logic.mjs from mse_loss.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 6dac9977868ce7ef5248582376372fdfb66261c8e2ced313e57aa48069b48cf0
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-mse/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABFANgAAF9YAR9fX1/AX1gA319fQF9AwQDAAECBAUBcAEBAQUDAQARBgkBfwFBgIDAAAsHOQQGbWVtb3J5AgANdDI3X21zZV9lbXB0eQAAD3QyN19tc2VfZ3JhZF9hdAABCnQyN19tc2VfYXQAAgqoAgMHAEMAAAAAC5sBAQJ/I4CAgIAAQSBrIgQkgICAgABBACEFIARBGGpBADYCACAEQgA3AxAgBCACOAIMIAQgATgCCCAEIAA4AgQCQANAIAVBDEYNASAEQRBqIAVqIARBBGogBWoqAgAgBUGAgMCAAGoqAgCTQ6uqKj+UOAIAIAVBBGohBQwACwsgBEEQaiADQQJ0aioCACECIARBIGokgICAgAAgAguAAQECfyOAgICAAEEQayIDJICAgIAAIAMgAjgCDCADIAE4AgggAyAAOAIEQwAAAAAhAkEAIQQCQANAIARBDEYNASACIANBBGogBGoqAgAgBEGAgMCAAGoqAgCTIgEgAZSSIQIgBEEEaiEEDAALCyADQRBqJICAgIAAIAJDAABAQJULCxUBAEGAgMAACwwAAIA/AAAAQAAAQEA=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function mseAt(p0, p1, p2) {
+ return W.t27_mse_at(p0, p1, p2);
+}
+function mseGradAt(p0, p1, p2, index) {
+ return W.t27_mse_grad_at(p0, p1, p2, int(index, 0, 4294967295, "mse_grad_at.index"));
+}
+function mseEmpty() {
+ return W.t27_mse_empty();
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward"]);
+export {
+ __NOT_LOWERED__,
+ mseAt,
+ mseEmpty,
+ mseGradAt
+};
diff --git a/apps/website/public/widgets/nn-mse/logic.t27 b/apps/website/public/widgets/nn-mse/logic.t27
new file mode 100644
index 0000000000..f748e4e61b
--- /dev/null
+++ b/apps/website/public/widgets/nn-mse/logic.t27
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Penalizes large errors more heavily | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/mse_loss.tri: the port kept only
+// the first parameter and returned void, so forward computed a value and
+// threw it away, and every test asserted `result != undefined`.
+
+module MseLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ // reduction: "mean", "sum" or "none" (from the .tri). forward below is the
+ // .tri's mean; the field is carried, not yet read.
+ pub const MSELossConfig = struct {
+ reduction : []const u8,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(predictions, targets) -> mean((predictions - targets)^2); 0 for n = 0
+ fn forward(predictions: []const f32, targets: []const f32) -> f32 {
+ if (predictions.len == 0) {
+ return 0.0;
+ }
+ var sum_loss : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predictions.len) {
+ const diff = predictions[i] - targets[i];
+ sum_loss = sum_loss + diff * diff;
+ i = i + 1;
+ }
+ return sum_loss / (predictions.len as f32);
+ }
+
+ // backward: grad_input[i] = 2 (predictions[i] - targets[i]) / n
+ fn backward(predictions: []const f32, targets: []const f32, grad_input: []f32) -> void {
+ if (predictions.len == 0) {
+ return;
+ }
+ const scale = 2.0 / (predictions.len as f32);
+ var i : usize = 0;
+ while (i < predictions.len) {
+ grad_input[i] = scale * (predictions[i] - targets[i]);
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ fn mse_at(p0: f32, p1: f32, p2: f32) -> f32 {
+ const predictions : [3]f32 = [p0, p1, p2];
+ const targets : [3]f32 = [1.0, 2.0, 3.0];
+ return forward(&predictions, &targets);
+ }
+
+ fn mse_grad_at(p0: f32, p1: f32, p2: f32, index: usize) -> f32 {
+ const predictions : [3]f32 = [p0, p1, p2];
+ const targets : [3]f32 = [1.0, 2.0, 3.0];
+ var grads : [3]f32 = [0.0] ** 3;
+ backward(&predictions, &targets, &grads);
+ return grads[index];
+ }
+
+ fn mse_empty() -> f32 {
+ const none : [0]f32 = [];
+ return forward(&none, &none);
+ }
+
+ test forward_zero_on_exact_match
+ given result = mse_at(1.0, 2.0, 3.0)
+ when also = mse_empty()
+ then result == 0.0 and also == 0.0
+
+ // errors (0.5, -1, 2): (0.25 + 1 + 4) / 3 = 1.75
+ test forward_basic_case
+ given result = mse_at(1.5, 1.0, 5.0)
+ when flipped = mse_at(0.5, 3.0, 1.0)
+ then @abs(result - 1.75) < 0.000001 and @abs(flipped - result) < 0.000001
+
+ test backward_matches_finite_difference
+ given slope = (mse_at(1.5, 1.001, 5.0) - mse_at(1.5, 0.999, 5.0)) / 0.002
+ when result = mse_grad_at(1.5, 1.0, 5.0, 1)
+ then @abs(result - slope) < 0.001 and @abs(result - (-2.0 / 3.0)) < 0.000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant mse_zero_at_target
+ assert mse_at(1.0, 2.0, 3.0) == 0.0
diff --git a/apps/website/public/widgets/nn-mse/nn-mse.t27 b/apps/website/public/widgets/nn-mse/nn-mse.t27
new file mode 100644
index 0000000000..1f045e1fbd
--- /dev/null
+++ b/apps/website/public/widgets/nn-mse/nn-mse.t27
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-mse.t27 -- mean squared error: how wrong a guess is, as one number
+; Source of truth for public/widgets/nn-mse/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-mse/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/loss/mse_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 13 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader moves three predictions from -1 to 5. The loss is mse_at(p0, p1, p2)
+; against the spec's targets (1, 2, 3), each gradient mse_grad_at, and the bowl mse_at over p0.
+
+; WHAT IT DOES NOT CLAIM: the targets are the spec's fixture, repeated as K_TARGETS; every loss and
+; gradient comes from the compiled fn.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_mse;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-mse";
+pub const TITLE : str = "Mean squared error: how wrong a guess is, as one number";
+pub const DESCRIPTION : str = "Move three predictions against the targets 1, 2 and 3: the loss is the mean of the squared misses, its bowl is a parabola, and each gradient points its prediction back at the target.";
+pub const IMAGE_ALT : str = "MSE card: predictions 1.5, 2 and 2 against targets 1, 2, 3 drawn as white rings with green dots, red dashed misses and pink gradient arrows. Below, the loss as prediction 0 moves is a gold parabola with its cursor at 1.5. Tiles read loss 0.4167, gradients 0.333 0.000 -0.667, loss at the targets 0.0000 and misses of 1 and 2 costing 0.333 and 1.333.";
+pub const HOOK : str = "A miss of 1 costs 1 / 3 of the mean; a miss of 2 costs 4 / 3. Squaring is what makes MSE fear large errors and shrug at small ones.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/loss/mse_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 13 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_TARGETS : [3]u8 = [1, 2, 3];
+pub const K_P_LO_C : i16 = -100;
+pub const K_P_HI_C : u16 = 500;
+pub const K_P_TICKS : [7]i8 = [-1, 0, 1, 2, 3, 4, 5];
+pub const K_LOSS_HI_C : u16 = 600;
+pub const K_LOSS_TICKS : [4]u8 = [0, 2, 4, 6];
+pub const K_PRESET_P0_C : [4]i16 = [150, 100, 300, -100];
+pub const K_PRESET_P1_C : [4]u16 = [200, 200, 200, 400];
+pub const K_PRESET_P2_C : [4]u16 = [200, 300, 300, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["1.5, 2, 2", "On target", "One miss of 2", "All wrong"];
+pub const SAY_ASK : str = "How wrong is a guess, as one number?";
+pub const SAY_PRED : str = "prediction";
+pub const SAY_TARGET : str = "target";
+pub const SAY_GRAD : str = "minus the gradient";
+pub const SAY_MISS_LABEL : str = "Each prediction, its target and the step the gradient asks for";
+pub const SAY_BOWL_LABEL : str = "The loss as prediction 0 moves, the others held";
+pub const SAY_BOWL : str = "Drag along the bowl: the loss as prediction 0 alone moves.";
+pub const SAY_LOSS : str = "loss";
+pub const SAY_GRADS : str = "gradients";
+pub const SAY_AT_TARGET : str = "loss on target";
+pub const SAY_DOUBLE : str = "miss of 1 / of 2";
+pub const SAY_NOTE : str = "The gradient of each prediction is 2 (p - t) / n: the further off, the harder the pull. Gradient descent follows it downhill into the bottom of the bowl.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/loss/mse_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_targets_are_the_spec_fixture {
+ assert K_TARGETS[0] == 1;
+ assert K_TARGETS[1] == 2;
+ assert K_TARGETS[2] == 3;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_P0_C[3] >= K_P_LO_C;
+ assert K_PRESET_P1_C[3] <= K_P_HI_C;
+ assert K_PRESET_P0_C[1] == K_TARGETS[0] * 100;
+}
diff --git a/apps/website/public/widgets/nn-mse/tool.js b/apps/website/public/widgets/nn-mse/tool.js
new file mode 100644
index 0000000000..d59ee9ac54
--- /dev/null
+++ b/apps/website/public/widgets/nn-mse/tool.js
@@ -0,0 +1,46 @@
+// nn-mse/tool.js -- lesson 13 of neural-networks: how wrong is a guess, as one number. Mean squared
+// error averages the squared misses, so a miss twice as far costs four times as much, and its
+// gradient points each prediction back at its target. Every value is mse_at or mse_grad_at of
+// gHashTag/t27 specs/ml/loss/mse_loss.t27, compiled to wasm (./logic.js), against the spec's targets
+// (1, 2, 3) (K_TARGETS, held equal by the widget's tests). Words are SAY_ and numbers K_ constants of
+// specs/widgets/nn-mse.t27.
+import { mseAt, mseGradAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const T = K('K_TARGETS', [])
+const P0 = centi(K('K_P_LO_C', -100)), P1 = centi(K('K_P_HI_C', 500))
+const pred = (key, i) => ({ key, label: `${say('SAY_PRED')} ${i}`, min: P0, max: P1, step: 0.05, fmt: (v) => fmt(v, 2) })
+const ctl = controls([pred('p0', 0), pred('p1', 1), pred('p2', 2)],
+ { names: W.SAY_PRESET_NAMES, values: { p0: K('K_PRESET_P0_C', []).map(centi), p1: K('K_PRESET_P1_C', []).map(centi), p2: K('K_PRESET_P2_C', []).map(centi) } }, draw)
+
+const misses = canvas('nn-curve', say('SAY_MISS_LABEL'))
+const bowl = canvas('nn-curve', say('SAY_BOWL_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, misses, legend([[say('SAY_TARGET'), '#ffffff'], [say('SAY_PRED'), COLORS[0]], [say('SAY_GRAD'), COLORS[3]]]), el('p', { class: 'dk-note' }, say('SAY_BOWL')), bowl, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const p = [s.p0, s.p1, s.p2]
+ misses.replaceChildren()
+ const m = plot(misses, { w: 640, h: 300, x0: -0.5, x1: 2.5, y0: P0, y1: P1, xticks: [0, 1, 2], yticks: K('K_P_TICKS', []), xfmt: (i) => `${say('SAY_PRED')} ${i}`, left: 40 })
+ for (let i = 0; i < 3; i++) {
+ const g = mseGradAt(s.p0, s.p1, s.p2, i)
+ misses.append(svgEl('line', { x1: m.x(i), x2: m.x(i), y1: m.clampY(T[i]), y2: m.clampY(p[i]), stroke: COLORS[6], 'stroke-width': 3, 'stroke-dasharray': '4 4' }))
+ misses.append(svgEl('line', { x1: m.x(i) + 26, x2: m.x(i) + 26, y1: m.clampY(p[i]), y2: m.clampY(p[i] - g), stroke: COLORS[3], 'stroke-width': 4 }))
+ misses.append(svgEl('circle', { cx: m.x(i), cy: m.clampY(T[i]), r: 8, fill: 'none', stroke: '#fff', 'stroke-width': 3 }))
+ misses.append(svgEl('circle', { cx: m.x(i), cy: m.clampY(p[i]), r: 9, fill: COLORS[0] }))
+ }
+ bowl.replaceChildren()
+ const o = { w: 640, h: 280, x0: P0, x1: P1, y0: 0, y1: centi(K('K_LOSS_HI_C', 600)), xticks: K('K_P_TICKS', []), yticks: K('K_LOSS_TICKS', []), xlabel: `${say('SAY_PRED')} 0` }
+ const b = plot(bowl, o)
+ b.line(sample((x) => mseAt(x, s.p1, s.p2), P0, P1), COLORS[1], 3)
+ cursor(bowl, b, s.p0, [[mseAt(s.p0, s.p1, s.p2), COLORS[1]]])
+ dragX(bowl, b, o, (x) => ctl.set('p0', x))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_LOSS'), value: fmt(mseAt(s.p0, s.p1, s.p2), 4), tone: 1 },
+ { label: say('SAY_GRADS'), value: [0, 1, 2].map((i) => fmt(mseGradAt(s.p0, s.p1, s.p2, i), 3)).join(' '), tone: 3 },
+ { label: say('SAY_AT_TARGET'), value: fmt(mseAt(T[0], T[1], T[2]), 4), tone: 0 },
+ { label: say('SAY_DOUBLE'), value: `${fmt(mseAt(T[0] + 1, T[1], T[2]), 3)} / ${fmt(mseAt(T[0] + 2, T[1], T[2]), 3)}`, tone: 6 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-positions/card.png b/apps/website/public/widgets/nn-positions/card.png
new file mode 100644
index 0000000000..1b8679d659
Binary files /dev/null and b/apps/website/public/widgets/nn-positions/card.png differ
diff --git a/apps/website/public/widgets/nn-positions/index.html b/apps/website/public/widgets/nn-positions/index.html
new file mode 100644
index 0000000000..84f33234e6
--- /dev/null
+++ b/apps/website/public/widgets/nn-positions/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Positional encoding: how a transformer knows word order · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Positional encoding: how a transformer knows word order
+
Every position gets 8 numbers, sines and cosines at frequencies falling by 10 per pair. Drag the position: the fast clock spins, the slow ones barely move, and the stripes show all 48 rows.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-positions/logic.js b/apps/website/public/widgets/nn-positions/logic.js
new file mode 100644
index 0000000000..71c5913779
--- /dev/null
+++ b/apps/website/public/widgets/nn-positions/logic.js
@@ -0,0 +1,28 @@
+// Generated by scripts/t27-logic.mjs from positional_encoding.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 2c979472dc128223ad4645c13b2776435ddf344ea29ffa5ed2e6c0b0056d3f09
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-positions/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABLAdgAn9/AX1gBX9/f39/AX9gAnx/AXxgAn1/AX9gAXwBfWABfQF9YAJ9fwF9AwwLAAABAgMEBAUFBgUEBQFwAQEBBQMBABEGCQF/AUGAgMAACwciAwZtZW1vcnkCAAZ0MjdfcGUAAAx0MjdfaW52X2ZyZXEAAQqrJwvjAQICfwJ9I4CAgIAAQSBrIgIkgICAgABBACEDIAJBGGpBACkCmIDAgAA3AwAgAkEQakEAKQKQgMCAADcDACACQQhqQQApAoiAwIAANwMAIAJBACkCgIDAgAA3AwAgALMhBCACIQACQANAIANBCEYNASADQQZxs0MAAAC+lEOOXRNBlBCIgICAACAElCEFAkACQCADQQFxDQAgBRCHgICAACEFDAELIAUQioCAgAAhBQsgACAFOAIAIABBBGohACADQQFqIQMMAAsLIAIgAUECdGoqAgAhBSACQSBqJICAgIAAIAULGAAgAEEBdLMgAbOVQ45dE8GUEIiAgIAAC4oSBgp/AXwIfwF8AX8BfCOAgICAAEGwBGsiBSSAgICAACAEQQJ0KAKggMCAACIGIANBf2oiB2oiCEF/IAhBf0obQQFqIQggAkF9aiIJIAnBQRhtIglBGGxrwUEPdiAJasEiCUEAIAlBAEobIgogB2shCSAKQQJ0IANBAnRrQbSAwIAAaiELIAVB0ABqIQwDfwJAAkACQCAIDQAgA0EDdCAFQdAAampBeGohDSAGQX8gBkF/ShtBAWohDkEAIQsMAQsCQCAJQX9KDQBEAAAAAAAAAAAhDwwCCyALKAIAtyEPDAELA0ACQAJAIAsgDkYNAEQAAAAAAAAAACEPIAAhCSANIQwgByEIA0AgCEF/Rg0CIAhBf2ohCCAPIAkrAwAgDCsDAKKgIQ8gCUEIaiEJIAxBeGohDAwACwtB////A0H///8BIApBaGwgAmoiEEFoaiICQQFGGyERIAVBfGoiEiAGQQJ0aiETIApBAnRBsIDAgABqIQpBLyAQayEUQTAgEGshFSAFQZADakF4aiEWIBBBGEohFyAGIQwCQANAIBYgDEEDdCIIaiEJIAVBkANqIAhqKwMAIQ8gBSELIAwhCAJAA0AgCEUNASALIA8gD0QAAAAAAABwPqL8ArciGEQAAAAAAABwwaKg/AI2AgAgC0EEaiELIAhBf2ohCCAJKwMAIBigIQ8gCUF4aiEJDAALCyAPIAIQg4CAgAAiDyAPRAAAAAAAAMA/opxEAAAAAAAAIMCioCIPIA/8AiIOt6EhDwJAAkACQAJAIBcNAAJAIAINACASIAxBAnRqKAIAQRd1IRkMAgtBACEZIA9EAAAAAAAA4D9mRQ0DQQIhGQwCCyASIAxBAnRqIgggCCgCACIIIAggFXUiCCAVdGsiCTYCACAJIBR1IRkgCCAOaiEOCyAZQQBMDQELQQAhDSAFIQggDCEJA0ACQAJAAkAgCUUNACAIKAIAIQsCQCANRQ0AQf///wchDQwCCwJAIAtFDQBBgICACCENDAILQQAhDQwCCwJAIBdFDQAgEiAMQQJ0aiIIIAgoAgAgEXE2AgALIA5BAWohDiAZQQJHDQNEAAAAAAAA8D8gD6EhD0ECIRkgDUUNAyAPRAAAAAAAAPA/IAIQg4CAgAChIQ8MAwsgCCANIAtrNgIAQQEhDQsgCEEEaiEIIAlBf2ohCQwACwsCQCAPRAAAAAAAAAAAYg0AIBIgDEECdGohCUEAIQsgDCEIAkADQCAIIAZMDQEgCEF/aiEIIAkoAgAgC3IhCyAJQXxqIQkMAAsLIBMhCCAMIQcCQCALRQ0AIAUgDEECdGpBfGohCANAIAxBf2ohDCACQWhqIQIgCCgCACEJIAhBfGohCCAJRQ0ADAQLCwNAIAdBAWohByAIKAIAIQkgCEF8aiEIIAlFDQALIAwgByAMIAdLGyEOIAVB0ABqIAMgDGpBA3RqIQ0DQCAMQQFqIQsCQCAMIA5HDQAgByEMDAMLIAVB0ABqIAwgA2pBA3RqIAogC0ECdGooAgC3OQMARAAAAAAAAAAAIQ8gDSEJIAAhDCADIQgCQANAIAhFDQEgCEF/aiEIIA8gDCsDACAJKwMAoqAhDyAJQXhqIQkgDEEIaiEMDAALCyAFQZADaiALQQN0aiAPOQMAIA1BCGohDSALIQwMAAsLCwJAIA9BGCAQaxCDgICAACIPRAAAAAAAAHBBZkUNACAFIAxBAWoiCEECdGogD0QAAAAAAABwPqL8AiIJNgIAIAUgDEECdGogDyAJt0QAAAAAAABwwaKg/AI2AgAgCCEMIBAhAgwBCyAFIAxBAnRqIA/8AjYCAAsgBSAMQQJ0aiEJIAVBkANqIAxBA3RqIQtEAAAAAAAA8D8gAhCDgICAACEPIAwhCAJAA0ACQCAIQX9KDQAgBUGQA2ogDEEDdGohCyAMIQ0DQCANQQBIDQMgBiAMIA1rIgcgBiAHSBsiCEF/IAhBf0obQQFqIQlEAAAAAAAAAAAhD0EAIQgCQANAIAlFDQEgDyAIQfiVwIAAaisDACALIAhqKwMAoqAhDyAIQQhqIQggCUF/aiEJDAALCyAFQfABaiAHQQN0aiAPOQMAIAtBeGohCyANQX9qIQ0MAAsLIAsgDyAJKAIAt6I5AwAgCUF8aiEJIAtBeGohCyAIQX9qIQggD0QAAAAAAABwPqIhDwwACwsCQAJAAkACQCAEDgQCAAABAgsgBUHwAWogDEEDdGohCUQAAAAAAAAAACEPIAwhCAJAA0AgCEEASA0BIAhBf2ohCCAPIAkrAwCgIQ8gCUF4aiEJDAALCyABIA+aIA8gGRs5AwAgDEEAIAxBAEobIQggBSsD8AEgD6EhDyAFQfABakEIaiEJAkADQCAIRQ0BIAhBf2ohCCAPIAkrAwCgIQ8gCUEIaiEJDAALCyABIA+aIA8gGRs5AwgMAgsgDEEDdCAFQfABampBeGohCCAMIQkDQAJAIAlBAEoNACAMQQN0IAVB8AFqakF4aiEIIAwhCQNAAkAgCUEBSg0AIAVB8AFqIAxBA3RqIQhEAAAAAAAAAAAhDwJAA0AgDEECSA0BIAxBf2ohDCAPIAgrAwCgIQ8gCEF4aiEIDAALCyAFKwPwASEYAkAgGQ0AIAEgGDkDACABIA85AxAgASAFKwP4ATkDCAwGCyABIBiaOQMAIAEgD5o5AxAgASAFKwP4AZo5AwgMBQsgCCAIKwMAIg8gCEEIaiILKwMAIhigIho5AwAgCyAYIA8gGqGgOQMAIAhBeGohCCAJQX9qIQkMAAsLIAggCCsDACIPIAhBCGoiCysDACIYoCIaOQMAIAsgGCAPIBqhoDkDACAIQXhqIQggCUF/aiEJDAALCyAFQfABaiAMQQN0aiEIRAAAAAAAAAAAIQ8CQANAIAxBAEgNASAMQX9qIQwgDyAIKwMAoCEPIAhBeGohCAwACwsgASAPmiAPIBkbOQMACyAFQbAEaiSAgICAACAOQQdxDwsgBUGQA2ogC0EDdGogDzkDACANQQhqIQ0gC0EBaiELDAALCyAMIA85AwAgDEEIaiEMIAhBf2ohCCALQQRqIQsgCUEBaiEJDAALC9YCAgN+An8CQAJAIAAgAGINACAAvSICQv///////////wCDQoCAgICAgID4/wBaDQAgAkKAgICAgICAgIB/gyEDQQsgAkIBhiIEeadrIARCNYinIARCgICAgICAgBBUGyEFAkAgAUEASA0AAkAgAUH+DyAFa0wNAEQAAAAAAADwfyAApg8LAkAgBSABaiIGQQBKDQAgAiABrYYgA4S/DwsCQCAFQQBKDQAgAkEBIAVrrYZC/////////weDIAatQjSGhCADhL8PCyABrUI0hiACfL8PC0EAIAFrIQYgAUEAIAVrSg0BAkAgAUFMIAVrTg0AIAO/DwtCgICAgICAgAhCACAFQQBKGyACQv////////8Hg4QgBkEBIAEgBWprIgEgBiABSBtBf2pBP3GtIgSIQgF8QgGIIAJ6IARRrUJ/hYO/IACmIQALIAAPCyACIAatQjSGfb8LkQMDA38DfAF/I4CAgIAAQRBrIgIkgICAgAACQAJAIAC8IgNB/////wdxIgRB2p+k7gRLDQAgASAAuyIFIAVEg8jJbTBf5D+iRAAAAAAAADhDoEQAAAAAAAA4w6AiBkQAAABQ+yH5v6KgIAZEY2IaYbQQUb6ioCIHOQMAIAb8AiEEAkAgB0QAAABg+yHpv2NFDQAgASAFIAZEAAAAAAAA8L+gIgZEAAAAUPsh+b+ioCAGRGNiGmG0EFG+oqA5AwAgBEF/aiEEDAILIAdEAAAAYPsh6T9kRQ0BIAEgBSAGRAAAAAAAAPA/oCIGRAAAAFD7Ifm/oqAgBkRjYhphtBBRvqKgOQMAIARBAWohBAwBCwJAIARB////+wdLDQAgAiAEIARBF3ZB6n5qIghBF3Rrvrs5AwAgAiACQQhqIAhBAUEAEIKAgIAAIQQgAisDCCEGAkAgA0F/Sg0AIAEgBpo5AwBBACAEayEEDAILIAEgBjkDAAwBCyABIAAgAJO7OQMAQQAhBAsgAkEQaiSAgICAACAEC0sBAnwgACAAIACiIgGiIgIgASABoqIgAUSnRjuMh83GPqJEdOfK4vkAKr+goiAAIAIgAUSy+26JEBGBP6JEd6zLVFVVxb+goqCgtgtPAQF8IAAgAKIiAESBXgz9///fv6JEAAAAAAAA8D+gIAAgAKIiAURCOgXhU1WlP6KgIAAgAaIgAERpUO7gQpP5PqJEJx4P6IfAVr+goqC2C8kDAgN/AXwjgICAgABBEGsiASSAgICAAAJAAkAgALwiAkH/////B3EiA0Han6T6A0sNACADQYCAgMwDSQ0BIAC7EIWAgIAAIQAMAQsCQAJAAkACQCADQdKn7YMESQ0AIANB1uOIhwRPDQMgALshBCADQeDbv4UETw0BIAJBf0oNAiAERNIhM3982RJAoBCGgICAACEADAQLIAC7IQQCQCADQeSX24AESQ0ARBgtRFT7IQnARBgtRFT7IQlAIAJBf0obIASgmhCFgICAACEADAQLAkAgAkF/Sg0AIAREGC1EVPsh+T+gEIaAgIAAjCEADAQLIAREGC1EVPsh+b+gEIaAgIAAIQAMAwtEGC1EVPshGcBEGC1EVPshGUAgAkF/ShsgBKAQhYCAgAAhAAwCCyAERNIhM3982RLAoBCGgICAAIwhAAwBCwJAIANB////+wdLDQAgACABQQhqEISAgIAAIQMgASsDCCEEAkACQAJAAkAgA0EDcQ4EAAECAwALIAQQhYCAgAAhAAwECyAEEIaAgIAAIQAMAwsgBJoQhYCAgAAhAAwCCyAEEIaAgIAAjCEADAELIAAgAJMhAAsgAUEQaiSAgICAACAAC9QCAgN/A30CQAJAIAAgAFwNACAAvCIBQR92IQICQAJAAkACQAJAAkACQCABQf////8HcSIDQc/YupUESw0AIANBmOTF9QNNDQEgA0GSq5T8A00NAgwECyADQYCAgPwHSw0GAkAgAUEASA0AIANBmOTFlQRJDQAgAEMAAAB/lA8LIAFBf0oNA0MAAAAAIQQgA0G047+WBEsNBwwDCyADQYCAgMgDTQ0BQQAhA0MAAAAAIQUgACEEDAQLIAJBAXMgAmshAwwCCyAAQwAAgD+SDwsgAEM7qrg/lCACQQJ0KgK4lsCAAJL8ACEDCyAAIAOyIgRDAHIxv5SSIgAgBEOOvr81lCIFkyEECyAAIAQgBCAEIASUIgYgBkMVUjW7lEOPqio+kpSTIgaUQwAAAEAgBpOVIAWTkkMAAIA/kiEEIANFDQEgBCADEImAgIAADwsgACEECyAEC6YCAQR/AkACQCAAIABcDQAgALwiAkH/////B3FBgICA/AdPDQAgAkGAgICAeHEhA0EIIAJBAXQiBGdrIARBGHYgBEGAgIAISRshBAJAIAFBAEgNAAJAIAFB/gEgBGtMDQBDAACAfyAAmA8LAkAgBCABaiIFQQBKDQAgAiABdCADcr4PCwJAIARBAEoNACACQQEgBGt0Qf///wNxIAVBF3RyIANyvg8LIAFBF3QgAmq+DwsgAUEAIARrSg0BAkAgAUFpIARrTg0AIAO+DwtBgICABEEAIARBAEobIAJB////A3FyQQAgAWsiA0EBIAEgBGprIgEgAyABSBtBf2oiAXZBAWpBAXYgAmggAUEfcUZBf3NxviAAmCEACyAADwsgAUEXdCACar4LzgMDA38BfQF8I4CAgIAAQRBrIgEkgICAgAACQAJAIAC8IgJB/////wdxIgNB2p+k+gNLDQBDAACAPyEEIANBgICAzANJDQEgALsQhoCAgAAhBAwBCwJAAkACQAJAIANB0qftgwRJDQAgA0HW44iHBE8NAiADQd/bv4UETQ0BRBgtRFT7IRnARBgtRFT7IRlAIAJBf0obIAC7oBCGgICAACEEDAQLIAC7IQUgA0Hjl9uABEsNAgJAIAJBf0oNACAFRBgtRFT7Ifk/oBCFgICAACEEDAQLRBgtRFT7Ifk/IAWhEIWAgIAAIQQMAwsCQCACQX9KDQBE0iEzf3zZEsAgALuhEIWAgIAAIQQMAwsgALtE0iEzf3zZEsCgEIWAgIAAIQQMAgsCQCADQf////sHSw0AIAAgAUEIahCEgICAACEDIAErAwghBQJAAkACQAJAIANBA3EOBAABAgMACyAFEIaAgIAAIQQMBQsgBZoQhYCAgAAhBAwECyAFEIaAgIAAjCEEDAMLIAUQhYCAgAAhBAwCCyAAIACTIQQMAQtEGC1EVPshCcBEGC1EVPshCUAgAkF/ShsgBaAQhoCAgACMIQQLIAFBEGokgICAgAAgBAsLyhYBAEGAgMAAC8AWAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEDAAAABAAAAAQAAAAGAAAAg/miAERObgD8KRUA0VcnAN009QBi28AAPJmVAEGQQwBjUf4Au96rALdhxQA6biQA0k1CAEkG4AAJ6i4AHJLRAOsd/gApsRwA6D6nAPU1ggBEuy4AnOmEALQmcABBfl8A1pE5AFODOQCc9DkAi1+EACj5vQD4HzsA3v+XAA+YBQARL+8AClqLAG0fbQDPfjYACcsnAEZPtwCeZj8ALepfALondQDl68cAPXvxAPc5BwCSUooA+2vqAB+xXwAIXY0AMANWAHv8RgDwq2sAILzPADb0mgDjqR0AXmGRAAgb5gCFmWUAoBRfAI1AaACA2P8AJ3NNAAYGMQDKVhUAyahzAHviYABrjMAAGcRHAM1nwwAJ6NwAWYMqAIt2xACmHJYARK/dABlX0QClPgUABQf/ADN+PwDCMugAmE/eALt9MgAmPcMAHmvvAJ/4XgA1HzoAf/LKAPGHHQB8kCEAaiR8ANVu+gAwLXcAFTtDALUUxgDDGZ0ArcTCACxNQQAMAF0Ahn1GAONxLQCbxpoAM2IAALTSfAC0p5cAN1XVANc+9gCjEBgATXb8AGSdKgBw16sAY3z4AHqwVwAXFecAwElWADvW2QCnhDgAJCPLANaKdwBaVCMAAB+5APEKGwAZzt8AnzH/AGYeagCZV2EArPtHAH5/2AAiZbcAMuiJAOa/YADvxM0AbDYJAF0/1AAW3tcAWDveAN6bkgDSIigAKIboAOJYTQDGyjIACOMWAOB9ywAXwFAA8x2nABjgWwAuEzQAgxJiAINIAQD1jlsArbB/AB7p8gBISkMAEGfTAKrd2ACuX0IAamHOAAoopADTmbQABqbyAFx3fwCjwoMAYTyIAIpzeACvjFoAb9e9AC2mYwD0v8sAjYHvACbBZwBVykUAytk2ACio0gDCYY0AEsl3AAQmFAASRpsAxFnEAMjFRABNspEAABfzANRDrQApSeUA/dUQAAC+/AAelMwAcM7uABM+9QDs8YAAs+fDAMf4KACTBZQAwXE+AC4JswALRfMAiBKcAKsgewAutZ8AR5LCAHsyLwAMVW0AcqeQAGvnHwAxy5YAeRZKAEF54gD034kA6JSXAOLmhACZMZcAiO1rAF9fNgC7/Q4ASJq0AGekbABxckIAjV0yAJ8VuAC85QkAjTElAPd0OQAwBRwADQwBAEsIaAAs7lgAR6qQAHTnAgC91iQA932mAG5IcgCfFu8AjpSmALSR9gDRU1EAzwryACCYMwD1S34AsmNoAN0+XwBAXQMAhYl/AFVSKQA3ZMAAbdgQADJIMgBbTHUATnHUAEVUbgALCcEAKvVpABRm1QAnB50AXQRQALQ72wDqdsUAh/kXAElrfQAdJ7oAlmkpAMbMrACtFFQAkOJqAIjZiQAsclAABKS+AHcHlADzMHAAAPwnAOpxqABmwkkAZOA9AJfdgwCjP5cAQ5T9AA2GjAAxQd4AkjmdAN1wjAAXt+cACN87ABU3KwBcgKAAWoCTABARkgAP6NgAbICvANv/SwA4kA8AWRh2AGKlFQBhy7sAx4m5ABBAvQDS8gQASXUnAOu29gDbIrsAChSqAIkmLwBkg3YACTszAA6UGgBROqoAHaPCAK/trgBcJhIAbcJNAC16nADAVpcAAz+DAAnw9gArQIwAbTGZADm0BwAMIBUA2MNbAPWSxADGrUsATsqlAKc3zQDmqTYAq5KUAN1CaAAZY94AdozvAGiLUgD82zcArqGrAN8VMQAArqEADPvaAGRNZgDtBbcAKWUwAFdWvwBH/zoAavm5AHW+8wAok98Aq4AwAGaM9gAEyxUA+iIGANnkHQA9s6QAVxuPADbNCQBOQukAE76kADMjtQDwqhoAT2WoANLBpQALPw8AW3jNACP5dgB7iwQAiRdyAMamUwBvbuIA7+sAAJtKWADE2rcAqma6AHbPzwDRAh0AsfEtAIyZwQDDrXcAhkjaAPddoADGgPQArPAvAN3smgA/XLwA0N5tAJDHHwAq27YAoyU6AACvmgCtU5MAtlcEACkttABLgH4A2genAHaqDgB7WaEAFhIqANy3LQD65f0Aidv+AIm+/QDkdmwABqn8AD6AcACFbhUA/Yf/ACg+BwBhZzMAKhiGAE296gCz568Aj21uAJVnOQAxv1sAhNdIADDfFgDHLUMAJWE1AMlwzgAwy7gAv2z9AKQAogAFbOQAWt2gACFvRwBiEtIAuVyEAHBhSQBrVuAAmVIBAFBVNwAe1bcAM/HEABNuXwBdMOQAhS6pAB2ywwChMjYACLekAOqx1AAW9yEAj2nkACf/dwAMA4AAjUAtAE/NoAAgpZkAs6LTAC9dCgC0+UIAEdrLAH2+0ACb28EAqxe9AMqigQAIalwALlUXACcAVQB/FPAA4QeGABQLZACWQY0Ah77eANr9KgBrJbYAe4k0AAXz/gC5v54AaGpPAEoqqABPxFoALfi8ANdamAD0x5UADU2NACA6pgCkV18AFD+xAIA4lQDMIAEAcd2GAMnetgC/YPUATWURAAEHawCMsKwAssDQAFFVSAAe+w4AlXLDAKMGOwDAQDUABtx7AOBFzABOKfoA1srIAOjzQQB8ZN4Am2TYANm+MQCkl8MAd1jUAGnjxQDw2hMAujo8AEYYRgBVdV8A0r31AG6SxgCsLl0ADkTtABw+QgBhxIcAKf3pAOfW8wAifMoAb5E1AAjgxQD/140AbmriALD9xgCTCMEAfF10AGutsgDNbp0APnJ7AMYRagD3z6kAKXPfALXJugC3AFEA4rINAHS6JADlfWAAdNiKAA0VLACBGAwAfmaUAAEpFgCfenYA/f2+AFZF7wDZfjYA7NkTAIu6uQDEl/wAMagnAPFuwwCUxTYA2KhWALSotQDPzA4AEoktAG9XNAAsVokAmc7jANYguQBrXqoAPiqcABFfzAD9C0oA4fT7AI47bQDihiwA6dSEAPy0qQDv7tEALjXJAC85YQA4IUQAG9nIAIH8CgD7SmoALxzYAFO0hABOmYwAVCLMACpV3ADAxtYACxmWABpwuABplWQAJlpgAD9S7gB/EQ8A9LURAPzL9QA0vC0ANLzuAOhdzADdXmAAZ46bAJIz7wDJF7gAYVibAOFXvABRg8YA2D4QAN1xSAAtHN0ArxihACEsRgBZ89cA2XqYAJ5UwABPhvoAVgb8AOV5rgCJIjYAOK0iAGeT3ABV6KoAgiY4AMrnmwBRDaQAmTOxAKnXDgBpBUgAZbLwAH+IpwCITJcA+dE2ACGSswB7gkoAmM8hAECf3ADcR1UA4XQ6AGfrQgD+nd8AXtRfAHtnpAC6rHoAVfaiACuIIwBBulUAWW4IACEqhgA5R4MAiePmAOWe1ABJ+0AA/1bpABwPygDFWYoAlPorANPBxQAPxc8A21quAEfFhgCFQ2IAIYY7ACx5lAAQYYcAKkx7AIAsGgBDvxIAiCaQAHg8iQCoxOQA5dt7AMQ6wgAm9OoA92eKAA2SvwBloysAPZOxAL18CwCkUdwAJ91jAGnh3QCalBkAqCmVAGjOKAAJ7bQARJ8gAE6YygBwgmMAfnwjAA+5MgCn9Y4AFFbnACHxCAC1nSoAb35NAKUZUQC1+asAgt/WAJbdYQAWNgIAxDqfAIOioQBy7W0AOY16AIK4qQBrMlwARidbAAA07QDSAHcA/PRVAAFZTQDgcYAAAAAAQPsh+T8AAAAALUR0PgAAAICYRvg8AAAAYFHMeDsAAACAgxvwOQAAAEAgJXo4AAAAgCKC4zYAAAAAHfNpNQAAAD8AAAC/";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function invFreq(pair, dModel) {
+ return W.t27_inv_freq(int(pair, 0, 4294967295, "inv_freq.pair"), int(dModel, 0, 4294967295, "inv_freq.d_model"));
+}
+function pe(pos, dim) {
+ return W.t27_pe(int(pos, 0, 4294967295, "pe.pos"), int(dim, 0, 4294967295, "pe.dim"));
+}
+const __NOT_LOWERED__ = Object.freeze(["forward"]);
+export {
+ __NOT_LOWERED__,
+ invFreq,
+ pe
+};
diff --git a/apps/website/public/widgets/nn-positions/logic.t27 b/apps/website/public/widgets/nn-positions/logic.t27
new file mode 100644
index 0000000000..11ed097e9a
--- /dev/null
+++ b/apps/website/public/widgets/nn-positions/logic.t27
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Uses sin/cos at different frequencies for each dimension | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward read an undeclared `d_model`, called `math::pow` / `math::sin` /
+// `math::cos`, computed each encoding into a local and dropped it, and grew the
+// frequency with the dimension where Vaswani et al. (2017) shrink it. It now
+// writes the sinusoidal encoding
+// PE(pos, 2i) = sin(pos / 10000^(2i / d)), PE(pos, 2i + 1) = cos(pos / 10000^(2i / d))
+// for each position into a positions x d_model buffer.
+
+module PositionalEnc;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_MAX_LEN : u32 = 2048;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PosEncConfig = struct {
+ d_model : u32,
+ max_len : u32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // 1 / 10000^(2i / d) = exp(-(2i / d) ln 10000)
+ fn inv_freq(pair: u32, d_model: u32) -> f32 {
+ return exp(-(@as(f32, @floatFromInt(2 * pair)) / @as(f32, @floatFromInt(d_model))) * log(@as(f32, 10000.0)));
+ }
+
+ fn forward(positions: []const u32, config: PosEncConfig, out: []f32) -> void {
+ var p : usize = 0;
+ while (p < positions.len) {
+ const pos : f32 = @as(f32, @floatFromInt(positions[p]));
+ var dim : u32 = 0;
+ while (dim < config.d_model) {
+ const angle = pos * inv_freq(dim / 2, config.d_model);
+ if (dim % 2 == 0) {
+ out[p * config.d_model + dim] = sin(angle);
+ } else {
+ out[p * config.d_model + dim] = cos(angle);
+ }
+ dim = dim + 1;
+ }
+ p = p + 1;
+ }
+ }
+
+ // PE(pos, dim) for d_model 8
+ fn pe(pos: u32, dim: usize) -> f32 {
+ const positions : [1]u32 = [pos];
+ var out : [8]f32 = [9.0] ** 8;
+ forward(&positions, PosEncConfig{ .d_model = 8, .max_len = DEFAULT_MAX_LEN }, &out);
+ return out[dim];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // pos 1, d 8: dims 0/1 use frequency 1, dims 2/3 use 1/10 (10000^(2/8) = 10)
+ test forward_basic_case
+ given s0 = pe(1, 0)
+ when s2 = pe(1, 2)
+ then @abs(s0 - 0.8414710) < 0.000001 and @abs(pe(1, 1) - 0.5403023) < 0.000001 and @abs(s2 - 0.0998334) < 0.000001 and @abs(pe(1, 3) - 0.9950042) < 0.000001
+
+ // Position 0 encodes as (0, 1, 0, 1, ...)
+ test forward_simple
+ given s = pe(0, 6)
+ when c = pe(0, 7)
+ then s == 0.0 and c == 1.0
+
+ // Each sin/cos pair lies on the unit circle
+ test forward_pairs_are_unit_vectors
+ given p = pe(37, 2) * pe(37, 2) + pe(37, 3) * pe(37, 3)
+ then @abs(p - 1.0) < 0.000001
+
+ invariant positional_encoding_max_len_is_positive
+ assert DEFAULT_MAX_LEN > 0
diff --git a/apps/website/public/widgets/nn-positions/nn-positions.t27 b/apps/website/public/widgets/nn-positions/nn-positions.t27
new file mode 100644
index 0000000000..1d38dcee64
--- /dev/null
+++ b/apps/website/public/widgets/nn-positions/nn-positions.t27
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-positions.t27 -- sinusoidal positional encoding: stripes and clocks
+; Source of truth for public/widgets/nn-positions/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-positions/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/transformer/positional_encoding.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 25 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the position from 0 to 200. Every stripe, hand and value is pe(pos,
+; dim) of the spec at d_model 8; the speeds are inv_freq(pair, 8).
+
+; WHAT IT DOES NOT CLAIM: d_model 8 shows the pattern; a real model uses hundreds of dimensions, and
+; many now learn or rotate positions instead.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_positions;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-positions";
+pub const TITLE : str = "Positional encoding: how a transformer knows word order";
+pub const DESCRIPTION : str = "Every position gets 8 numbers, sines and cosines at frequencies falling by 10 per pair. Drag the position: the fast clock spins, the slow ones barely move, and the stripes show all 48 rows.";
+pub const IMAGE_ALT : str = "Positional-encoding card at position 5: four clocks with coloured hands, the fastest turned far round, the slowest still pointing up. Below, a stripe chart of 48 positions by 8 dimensions in pink and green: the left columns flicker fast, the right ones barely change. Tiles read the 8 values and the 4 speeds 1, 0.1, 0.01, 0.001.";
+pub const HOOK : str = "Position 5 is (-0.959, 0.284, 0.479, 0.878, 0.050, 0.999, 0.005, 1.000). Four clocks at speeds 1, 0.1, 0.01 and 0.001 give each position its own reading.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/transformer/positional_encoding.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 25 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_D_MODEL : u8 = 8;
+pub const K_ROWS : u8 = 48;
+pub const K_POS_HI : u8 = 200;
+pub const K_PRESET_POS : [4]u8 = [5, 0, 1, 37];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Position 5", "Position 0", "Position 1", "Position 37"];
+pub const SAY_ASK : str = "Attention sees a set of words. How does it know their order?";
+pub const SAY_POS : str = "position";
+pub const SAY_STRIPES_LABEL : str = "pe(pos, dim) for positions 0 to 47 (rows) and the 8 dimensions";
+pub const SAY_CLOCKS_LABEL : str = "Each sin and cos pair as a clock hand, and its speed";
+pub const SAY_PAIR : str = "pair";
+pub const SAY_VECTOR : str = "the 8 numbers";
+pub const SAY_FAST : str = "fastest pair";
+pub const SAY_SLOW : str = "slowest pair";
+pub const SAY_FREQS : str = "speeds";
+pub const SAY_NOTE : str = "A shift by k positions turns every clock by its own fixed angle, so the distance between two positions is visible to a dot product. Position 0 is (0, 1, 0, 1, ...).";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/transformer/positional_encoding.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test eight_dimensions_four_clocks {
+ assert K_D_MODEL == 8;
+ assert K_D_MODEL % 2 == 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_POS[3] < K_ROWS;
+ assert K_PRESET_POS[1] == 0;
+ assert K_POS_HI > K_ROWS;
+}
diff --git a/apps/website/public/widgets/nn-positions/tool.js b/apps/website/public/widgets/nn-positions/tool.js
new file mode 100644
index 0000000000..d78debd144
--- /dev/null
+++ b/apps/website/public/widgets/nn-positions/tool.js
@@ -0,0 +1,42 @@
+// nn-positions/tool.js -- lesson 25 of neural-networks: how a transformer knows the order of words.
+// Each position gets a vector of sines and cosines at frequencies falling by 10000^(2i/d): the fast
+// dimensions flicker, the slow ones barely move, and together every position gets its own pattern.
+// Every stripe, hand and tile is pe(pos, dim) of gHashTag/t27 specs/ml/transformer/
+// positional_encoding.t27, compiled to wasm (./logic.js), d_model 8. Words are SAY_ and numbers K_
+// constants of specs/widgets/nn-positions.t27.
+import { invFreq, pe } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fmt, heat, root, say, source, svgEl, tiles, tone } from '../nn-kit.js'
+
+const D = K('K_D_MODEL', 8), ROWS = K('K_ROWS', 48), PMAX = K('K_POS_HI', 200)
+const ctl = controls([
+ { key: 'pos', label: say('SAY_POS'), min: 0, max: PMAX },
+], { names: W.SAY_PRESET_NAMES, values: { pos: K('K_PRESET_POS', []) } }, draw)
+
+const stripes = canvas('nn-heat', say('SAY_STRIPES_LABEL'))
+const clocks = canvas('nn-clocks', say('SAY_CLOCKS_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('p', { class: 'dk-label' }, say('SAY_CLOCKS_LABEL')), clocks, stat, el('p', { class: 'dk-label' }, say('SAY_STRIPES_LABEL')), stripes, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ // Stripes: positions down, dimensions across; the selected position's row is outlined when it is shown.
+ heat(stripes, { rows: ROWS, cols: D, value: (r, c) => pe(r, c), color: (v) => tone(v, 1), colLabels: [...Array(D)].map((_, c) => `${c % 2 ? 'cos' : 'sin'}${c >> 1}`), w: 640, cell: 76, cellH: 9, fmt: null, mark: (r) => (r === s.pos ? 'on' : '') })
+ // Clocks: each sin/cos pair is a hand on a circle, turning at its own speed.
+ clocks.replaceChildren()
+ clocks.setAttribute('viewBox', '0 0 640 190')
+ for (let k = 0; k < D / 2; k++) {
+ const cx = 80 + k * 160, cy = 92, R = 62
+ const sx = pe(s.pos, 2 * k), cs = pe(s.pos, 2 * k + 1)
+ clocks.append(svgEl('circle', { cx, cy, r: R, fill: '#020403', stroke: COLORS[k + 2], 'stroke-width': 2, opacity: 0.9 }))
+ clocks.append(svgEl('line', { x1: cx, y1: cy, x2: cx + sx * R, y2: cy - cs * R, stroke: COLORS[k + 2], 'stroke-width': 5, 'stroke-linecap': 'round' }))
+ clocks.append(svgEl('circle', { cx: cx + sx * R, cy: cy - cs * R, r: 7, fill: COLORS[k + 2] }))
+ clocks.append(svgEl('text', { x: cx, y: 182, class: 'dk-tick', 'text-anchor': 'middle' }, `${say('SAY_PAIR')} ${k}: ${fmt(invFreq(k, D), 4)}`))
+ }
+ stat.replaceChildren(tiles([
+ { label: say('SAY_VECTOR'), value: [...Array(D)].map((_, c) => fmt(pe(s.pos, c), 2)).join(' '), tone: 0 },
+ { label: say('SAY_FAST'), value: `${fmt(pe(s.pos, 0), 3)}, ${fmt(pe(s.pos, 1), 3)}`, tone: 2 },
+ { label: say('SAY_SLOW'), value: `${fmt(pe(s.pos, D - 2), 4)}, ${fmt(pe(s.pos, D - 1), 4)}`, tone: 3 },
+ { label: say('SAY_FREQS'), value: [...Array(D / 2)].map((_, k) => fmt(invFreq(k, D), 3)).join(' '), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-ppo/card.png b/apps/website/public/widgets/nn-ppo/card.png
new file mode 100644
index 0000000000..6cc7fc183c
Binary files /dev/null and b/apps/website/public/widgets/nn-ppo/card.png differ
diff --git a/apps/website/public/widgets/nn-ppo/index.html b/apps/website/public/widgets/nn-ppo/index.html
new file mode 100644
index 0000000000..d88a7cafb9
--- /dev/null
+++ b/apps/website/public/widgets/nn-ppo/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+PPO: how an agent learns from reward without lurching · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
PPO: how an agent learns from reward without lurching
+
Drag the probability ratio of a new policy over the old: PPO's objective follows r times the advantage until r moves more than eps the rewarded way, then it goes flat and stops pulling.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-ppo/logic.js b/apps/website/public/widgets/nn-ppo/logic.js
new file mode 100644
index 0000000000..372de11955
--- /dev/null
+++ b/apps/website/public/widgets/nn-ppo/logic.js
@@ -0,0 +1,43 @@
+// Generated by scripts/t27-logic.mjs from ppo_clip_loss.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: d0883efe249316ba0beddd1063ef00c8546c8c982d5636a79971f30a3a1bf855
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-ppo/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABJQZgBH19fX0BfWAAAX1gAX0BfWADfX19AX1gAn19AX1gAn1/AX0DDAsAAQIBAwMEBAQCBQQFAXABAQEFAwEAEQYJAX8BQYCAwAALB5gBCAZtZW1vcnkCAA50MjdfdG90YWxfbG9zcwAAD3QyN192YWxpZF9pbnB1dAABEXQyN192YWx1ZV9sb3NzX29mAAITdDI3X2VudHJvcHlfbG9zc19vZgADEnQyN19wb2xpY3lfbG9zc19vZgAEFXQyN19jbGlwcGVkX3N1cnJvZ2F0ZQAFEXQyN19jb21wdXRlX3JhdGlvAAYKuwkLDQAgAiAAIAEgA5SSkgsHAEPNzEw+C7ABAgJ/An0jgICAgABBEGsiASSAgICAACABQoCAgICEgIDgPzcCCCABIAA4AgRDAAAAACEAQQAhAgJAA0AgAkEMRg0BIAFBBGogAmoqAgAgAkGMgMCAAGoqAgAiA5NDzcxMvhCIgICAACEEIAAgAkGAgMCAAGoqAgAgAyAEQ83MTD6WkpMiAyADQwAAAD+UlJIhACACQQRqIQIMAAsLIAFBEGokgICAgAAgAEMAAEBAlQsHAEMK16O7C6EBAgJ/AX0jgICAgABBEGsiAySAgICAACADIAI4AgwgAyABOAIIIAMgADgCBEMAAAAAIQJBACEEAkADQCAEQQxGDQEgA0EEaiAEaioCACIBQ83MTD8QiICAgAAhACACIAEgBEGYgMCAAGoqAgAiBZQgBSAAQ5qZmT+WlBCHgICAAJIhAiAEQQRqIQQMAAsLIANBEGokgICAgAAgAkMAAEDAlQsuACAAIAGUIAEgAEMAAIA/IAKTEIiAgIAAIAJDAACAP5IQh4CAgACUEIeAgIAACw0AIAAgAZMQiYCAgAALRQEBfwJAIAAgAFwNAAJAIAEgAVsNACAADwsCQCABvCAAvCICc0F/Sg0AIAAgASACQQBIGw8LIAAgASAAIAFdGyEBCyABC0UBAX8CQCAAIABcDQACQCABIAFbDQAgAA8LAkAgAbwgALwiAnNBf0oNACABIAAgAkEASBsPCyABIAAgACABXRshAQsgAQvUAgIDfwN9AkACQCAAIABcDQAgALwiAUEfdiECAkACQAJAAkACQAJAAkAgAUH/////B3EiA0HP2LqVBEsNACADQZjkxfUDTQ0BIANBkquU/ANNDQIMBAsgA0GAgID8B0sNBgJAIAFBAEgNACADQZjkxZUESQ0AIABDAAAAf5QPCyABQX9KDQNDAAAAACEEIANBtOO/lgRLDQcMAwsgA0GAgIDIA00NAUEAIQNDAAAAACEFIAAhBAwECyACQQFzIAJrIQMMAgsgAEMAAIA/kg8LIABDO6q4P5QgAkECdCoCpIDAgACS/AAhAwsgACADsiIEQwByMb+UkiIAIARDjr6/NZQiBZMhBAsgACAEIAQgBCAElCIGIAZDFVI1u5RDj6oqPpKUkyIGlEMAAABAIAaTlSAFk5JDAACAP5IhBCADRQ0BIAQgAxCKgICAAA8LIAAhBAsgBAumAgEEfwJAAkAgACAAXA0AIAC8IgJB/////wdxQYCAgPwHTw0AIAJBgICAgHhxIQNBCCACQQF0IgRnayAEQRh2IARBgICACEkbIQQCQCABQQBIDQACQCABQf4BIARrTA0AQwAAgH8gAJgPCwJAIAQgAWoiBUEASg0AIAIgAXQgA3K+DwsCQCAEQQBKDQAgAkEBIARrdEH///8DcSAFQRd0ciADcr4PCyABQRd0IAJqvg8LIAFBACAEa0oNAQJAIAFBaSAEa04NACADvg8LQYCAgARBACAEQQBKGyACQf///wNxckEAIAFrIgNBASABIARqayIBIAMgAUgbQX9qIgF2QQFqQQF2IAJoIAFBH3FGQX9zcb4gAJghAAsgAA8LIAFBF3QgAmq+Cws1AQBBgIDAAAsszcyMP2ZmBkDNzMw/ZmZmPzMz8z8zM7M/AAAAQAAAwD8AAEBAAAAAPwAAAL8=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function computeRatio(newLogProb, oldLogProb) {
+ return W.t27_compute_ratio(newLogProb, oldLogProb);
+}
+function clippedSurrogate(ratio, advantage, clipEpsilon) {
+ return W.t27_clipped_surrogate(ratio, advantage, clipEpsilon);
+}
+function policyLossOf(r0, r1, r2) {
+ return W.t27_policy_loss_of(r0, r1, r2);
+}
+function entropyLossOf() {
+ return W.t27_entropy_loss_of();
+}
+function valueLossOf(p0) {
+ return W.t27_value_loss_of(p0);
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+function totalLoss(policyLossValue, valueLossValue, entropyLossValue, valueLossCoef) {
+ return W.t27_total_loss(policyLossValue, valueLossValue, entropyLossValue, valueLossCoef);
+}
+const __NOT_LOWERED__ = Object.freeze(["policy_loss", "entropy_loss", "value_loss"]);
+export {
+ __NOT_LOWERED__,
+ clippedSurrogate,
+ computeRatio,
+ entropyLossOf,
+ policyLossOf,
+ totalLoss,
+ validInput,
+ valueLossOf
+};
diff --git a/apps/website/public/widgets/nn-ppo/logic.t27 b/apps/website/public/widgets/nn-ppo/logic.t27
new file mode 100644
index 0000000000..72d30bb304
--- /dev/null
+++ b/apps/website/public/widgets/nn-ppo/logic.t27
@@ -0,0 +1,197 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Probability ratio between policies | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written with `let x = if ... {}` and `for i in 0..n`, so it never compiled.
+// Three formulas were wrong, and their tests pinned the wrong values:
+// - compute_ratio returned log pi_new - log pi_old; the ratio is exp of that.
+// - clipped_surrogate returned clip(r) A; PPO's term is min(r A, clip(r) A), so
+// r = 1.1 inside the clip range gives 2.2, not 2.4.
+// - policy_loss returned the surrogate itself, a quantity to MAXIMISE; as a loss
+// added to the value and entropy losses it is minus the mean surrogate.
+// Each test now states the value PPO defines (Schulman et al., 2017).
+
+module PpoClipLoss;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_CLIP_EPSILON : f32 = 0.2;
+ const DEFAULT_ENTROPY_COEF : f32 = 0.01;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const PPOClipConfig = struct {
+ clip_epsilon : f32,
+ entropy_coef : f32,
+ value_clip_coef : f32,
+ };
+
+ pub const PPOBatch = struct {
+ states : [][]f32,
+ actions : [][]f32,
+ old_log_probs : []f32,
+ advantages : []f32,
+ returns : []f32,
+ values : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // r = pi_new / pi_old = exp(log pi_new - log pi_old)
+ fn compute_ratio(new_log_prob: f32, old_log_prob: f32) -> f32 {
+ return exp(new_log_prob - old_log_prob);
+ }
+
+ // min(r A, clip(r, 1 - eps, 1 + eps) A)
+ fn clipped_surrogate(ratio: f32, advantage: f32, clip_epsilon: f32) -> f32 {
+ const clipped_ratio = @min(@max(ratio, 1.0 - clip_epsilon), 1.0 + clip_epsilon);
+ return @min(ratio * advantage, clipped_ratio * advantage);
+ }
+
+ // -mean_t min(r_t A_t, clip(r_t) A_t): the loss gradient descent minimises
+ fn policy_loss(ratios: []const f32, advantages: []const f32, clip_epsilon: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < ratios.len) {
+ total = total + clipped_surrogate(ratios[i], advantages[i], clip_epsilon);
+ i = i + 1;
+ }
+ return -total / @as(f32, @floatFromInt(ratios.len));
+ }
+
+ // -coef * mean entropy: lower loss for a more exploratory policy
+ fn entropy_loss(entropies: []const f32, entropy_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < entropies.len) {
+ total = total + entropies[i];
+ i = i + 1;
+ }
+ return -entropy_coef * (total / @as(f32, @floatFromInt(entropies.len)));
+ }
+
+ // mean 0.5 (target - (old + clip(pred - old, -c, c)))^2
+ fn value_loss(predicted_values: []const f32, target_values: []const f32, old_values: []const f32, clip_coef: f32) -> f32 {
+ var total : f32 = 0.0;
+ var i : usize = 0;
+ while (i < predicted_values.len) {
+ const diff = @min(@max(predicted_values[i] - old_values[i], -clip_coef), clip_coef);
+ const err = target_values[i] - (old_values[i] + diff);
+ total = total + 0.5 * err * err;
+ i = i + 1;
+ }
+ return total / @as(f32, @floatFromInt(predicted_values.len));
+ }
+
+ fn policy_loss_of(r0: f32, r1: f32, r2: f32) -> f32 {
+ const ratios : [3]f32 = [r0, r1, r2];
+ const advantages : [3]f32 = [2.0, 1.5, 3.0];
+ return policy_loss(&ratios, &advantages, DEFAULT_CLIP_EPSILON);
+ }
+
+ fn entropy_loss_of() -> f32 {
+ const entropies : [3]f32 = [0.5, 0.6, 0.4];
+ return entropy_loss(&entropies, DEFAULT_ENTROPY_COEF);
+ }
+
+ fn value_loss_of(p0: f32) -> f32 {
+ const predicted : [3]f32 = [p0, 2.0, 1.5];
+ const target : [3]f32 = [1.1, 2.1, 1.6];
+ const old : [3]f32 = [0.9, 1.9, 1.4];
+ return value_loss(&predicted, &target, &old, 0.2);
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_CLIP_EPSILON;
+ }
+
+ // total_loss(policy_loss_value: f32, value_loss_value: f32, entropy_loss_value: f32, value_loss_coef: f32) → f32
+ fn total_loss(policy_loss_value: f32, value_loss_value: f32, entropy_loss_value: f32, value_loss_coef: f32) -> f32 {
+ // Combine all loss components
+ return policy_loss_value + value_loss_coef * value_loss_value + entropy_loss_value;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test compute_ratio_basic_case
+ given
+ new_log_prob = 0.5
+ old_log_prob = 0.3
+ when result = compute_ratio(new_log_prob, old_log_prob)
+ then @abs(result - 1.2214028) < 0.000001 // exp(0.2); the old `== 0.2` was the log ratio
+
+ test clipped_surrogate_basic_case
+ given
+ ratio = 1.1
+ advantage = 2.0
+ clip_epsilon = 0.2
+ when result = clipped_surrogate(ratio, advantage, clip_epsilon)
+ then @abs(result - 1.1 * 2.0) < 0.000001 // inside [0.8, 1.2] the clip does nothing
+
+ test clipped_surrogate_clipping_case
+ given
+ ratio = 1.5 // Above clip threshold of 1.2
+ advantage = 2.0
+ clip_epsilon = 0.2
+ when result = clipped_surrogate(ratio, advantage, clip_epsilon)
+ then @abs(result - 1.2 * 2.0) < 0.000001 and clipped_surrogate(0.5, -1.0, 0.2) == -0.8
+
+ // min(r A, clip(r) A) = (2.0, 1.65, 2.7): the loss is -mean = -2.1166667. The old `> 0` was
+ // the surrogate's sign; a ratio pushed past the clip earns nothing more.
+ test policy_loss_basic_case
+ given result = policy_loss_of(1.0, 1.1, 0.9)
+ when pushed = policy_loss_of(1.0, 1.1, 3.0)
+ then result < 0.0 and @abs(result + 2.1166667) < 0.000001 and @abs(pushed - policy_loss_of(1.0, 1.1, 1.2)) < 0.000001
+
+ test entropy_loss_basic_case
+ given result = entropy_loss_of()
+ then result < 0.0 and @abs(result + 0.005) < 0.0000001
+
+ test value_loss_basic_case
+ given result = value_loss_of(1.0)
+ when clipped = value_loss_of(5.0)
+ then result > 0.0 and @abs(result - 0.005) < 0.0000001 and @abs(clipped - value_loss_of(1.1)) < 0.0000001
+
+ test total_loss_basic_case
+ given
+ policy_loss_value = 1.0
+ value_loss_value = 0.5
+ entropy_loss_value = -0.01
+ value_loss_coef = 0.5
+ when result = total_loss(policy_loss_value, value_loss_value, entropy_loss_value, value_loss_coef)
+ then @abs(result - (1.0 + 0.5 * 0.5 + (-0.01))) < 0.0000001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // 0 < clip_epsilon <= 1 (typically 0.1-0.3)
+ invariant ppo_clip_loss_constraint_0
+ assert valid_input() > 0.0 and valid_input() <= 1.0
+
+ // entropy_coef >= 0
+ invariant ppo_clip_loss_constraint_1
+ assert DEFAULT_ENTROPY_COEF >= 0.0
+
+ // name: ppo_objective -- for A > 0 the surrogate stops growing at r = 1 + eps
+ invariant ppo_clip_loss_constraint_2
+ assert clipped_surrogate(5.0, 1.0, 0.2) == 1.2
+
+ // name: probability_ratio -- equal log-probabilities give r = 1
+ invariant ppo_clip_loss_constraint_3
+ assert compute_ratio(0.3, 0.3) == 1.0
+
+ // "Schulman et al. (2017) - Proximal Policy Optimization Algorithms, arXiv:1707.06347":
+ // for A < 0 the surrogate stops falling at r = 1 - eps
+ invariant ppo_clip_loss_constraint_4
+ assert clipped_surrogate(0.1, -1.0, 0.2) == -0.8
diff --git a/apps/website/public/widgets/nn-ppo/nn-ppo.t27 b/apps/website/public/widgets/nn-ppo/nn-ppo.t27
new file mode 100644
index 0000000000..f32a2aeb05
--- /dev/null
+++ b/apps/website/public/widgets/nn-ppo/nn-ppo.t27
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-ppo.t27 -- PPO's clipped surrogate: a reward that stops pulling
+; Source of truth for public/widgets/nn-ppo/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-ppo/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/rl/ppo_clip_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 27 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags r from 0 to 2 and sets the advantage A from -2 to 2 and eps from
+; 0.05 to 0.5. The green curve is clipped_surrogate(r, A, eps), the dashed one clipped_surrogate(r,
+; A, 1000), which never clips here, and the tiles policy_loss_of on the spec's advantages (2, 1.5,
+; 3).
+
+; WHAT IT DOES NOT CLAIM: the objective of one sample; PPO averages it over a batch, adds value and
+; entropy terms (total_loss in the spec) and runs it for a few epochs per batch.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_ppo;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-ppo";
+pub const TITLE : str = "PPO: how an agent learns from reward without lurching";
+pub const DESCRIPTION : str = "Drag the probability ratio of a new policy over the old: PPO's objective follows r times the advantage until r moves more than eps the rewarded way, then it goes flat and stops pulling.";
+pub const IMAGE_ALT : str = "PPO card at r = 1.5, A = 1, eps 0.2: the dashed gold raw objective is a straight line through the origin; the thick green clipped one follows it and turns flat at 1.2 from r = 1.2 on. Tiles read clipped 1.2000, raw 1.5000, the spec batch policy loss -2.1167 and the same batch at r = 1.5, -2.6000.";
+pub const HOOK : str = "At r = 1.5 with advantage 1 and eps 0.2, the raw objective is 1.5 but PPO counts only 1.2. The clip is what keeps one update from breaking the policy.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/rl/ppo_clip_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 27 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_R_HI_C : u16 = 200;
+pub const K_A_LO_C : i16 = -200;
+pub const K_A_HI_C : u16 = 200;
+pub const K_EPS_LO_C : u8 = 5;
+pub const K_EPS_HI_C : u8 = 50;
+pub const K_WIDE : u16 = 1000;
+pub const K_Y_LO : i8 = -4;
+pub const K_Y_HI : u8 = 4;
+pub const K_Y_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_R_TICKS_C : [5]u8 = [0, 50, 100, 150, 200];
+pub const K_BATCH_R_C : [3]u8 = [100, 110, 90];
+pub const K_PRESET_R_C : [4]u8 = [150, 50, 150, 80];
+pub const K_PRESET_A_C : [4]i16 = [100, -100, 200, -200];
+pub const K_PRESET_EPS_C : [4]u8 = [20, 20, 50, 10];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["r 1.5, A 1", "r 0.5, A -1", "r 1.5, A 2, eps 0.5", "r 0.8, A -2, eps 0.1"];
+pub const SAY_ASK : str = "How does an agent learn from reward without lurching?";
+pub const SAY_R : str = "ratio r = new / old";
+pub const SAY_A : str = "advantage A";
+pub const SAY_EPS : str = "clip eps";
+pub const SAY_PLOT_LABEL : str = "PPO's clipped objective and the raw r A as the ratio goes from 0 to 2";
+pub const SAY_CLIPPED : str = "PPO objective";
+pub const SAY_RAW : str = "raw r A";
+pub const SAY_BATCH : str = "spec batch loss";
+pub const SAY_BATCH_FAR : str = "same batch at r";
+pub const SAY_NOTE : str = "A positive advantage says the action was better than expected, so raise its probability; past 1 + eps there is no more credit for it. A negative one is clipped at 1 - eps the same way.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/rl/ppo_clip_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_wide_clip_never_clips_the_window {
+ assert K_WIDE * 100 > K_R_HI_C;
+ assert K_EPS_HI_C < 100;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_R_C[0] <= K_R_HI_C;
+ assert K_PRESET_A_C[3] == K_A_LO_C;
+ assert K_PRESET_EPS_C[3] >= K_EPS_LO_C;
+ assert K_BATCH_R_C[0] == 100;
+}
diff --git a/apps/website/public/widgets/nn-ppo/tool.js b/apps/website/public/widgets/nn-ppo/tool.js
new file mode 100644
index 0000000000..327cc2a0ad
--- /dev/null
+++ b/apps/website/public/widgets/nn-ppo/tool.js
@@ -0,0 +1,40 @@
+// nn-ppo/tool.js -- lesson 27 of neural-networks: how an agent learns from reward without lurching.
+// PPO scores a change in policy by min(r A, clip(r, 1 - eps, 1 + eps) A): once the probability
+// ratio r has moved more than eps in the direction the advantage A rewards, the objective goes flat
+// and stops pulling. Both curves and every tile are clipped_surrogate of gHashTag/t27
+// specs/ml/rl/ppo_clip_loss.t27, compiled to wasm (./logic.js); the dashed curve is the same fn with
+// a clip range so wide (K_WIDE) it never clips. Words are SAY_ and numbers K_ constants of
+// specs/widgets/nn-ppo.t27.
+import { clippedSurrogate, policyLossOf } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const R0 = 0, R1 = centi(K('K_R_HI_C', 200))
+const WIDE = K('K_WIDE', 1000)
+const ctl = controls([
+ { key: 'r', label: say('SAY_R'), min: R0, max: R1, step: 0.01, fmt: (v) => fmt(v, 2) },
+ { key: 'a', label: say('SAY_A'), min: centi(K('K_A_LO_C', -200)), max: centi(K('K_A_HI_C', 200)), step: 0.1, fmt: (v) => fmt(v, 1) },
+ { key: 'eps', label: say('SAY_EPS'), min: centi(K('K_EPS_LO_C', 5)), max: centi(K('K_EPS_HI_C', 50)), step: 0.01, fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { r: K('K_PRESET_R_C', []).map(centi), a: K('K_PRESET_A_C', []).map(centi), eps: K('K_PRESET_EPS_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_CLIPPED'), COLORS[0]], [say('SAY_RAW'), COLORS[1], true]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 340, x0: R0, x1: R1, y0: K('K_Y_LO', -4), y1: K('K_Y_HI', 4), xticks: K('K_R_TICKS_C', []).map(centi), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_R') }
+ const p = plot(svg, o)
+ p.line(sample((r) => clippedSurrogate(r, s.a, WIDE), R0, R1), COLORS[1], 2, { 'stroke-dasharray': '7 6' })
+ p.line(sample((r) => clippedSurrogate(r, s.a, s.eps), R0, R1, 400), COLORS[0], 4)
+ cursor(svg, p, s.r, [[clippedSurrogate(s.r, s.a, s.eps), COLORS[0]], [clippedSurrogate(s.r, s.a, WIDE), COLORS[1]]])
+ dragX(svg, p, o, (r) => ctl.set('r', r))
+ const rs = K('K_BATCH_R_C', []).map(centi)
+ stat.replaceChildren(tiles([
+ { label: say('SAY_CLIPPED'), value: fmt(clippedSurrogate(s.r, s.a, s.eps), 4), tone: 0 },
+ { label: say('SAY_RAW'), value: fmt(clippedSurrogate(s.r, s.a, WIDE), 4), tone: 1 },
+ { label: say('SAY_BATCH'), value: fmt(policyLossOf(rs[0], rs[1], rs[2]), 4), tone: 2 },
+ { label: say('SAY_BATCH_FAR'), value: fmt(policyLossOf(s.r, s.r, s.r), 4), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-relu/card.png b/apps/website/public/widgets/nn-relu/card.png
new file mode 100644
index 0000000000..8f3f047b7e
Binary files /dev/null and b/apps/website/public/widgets/nn-relu/card.png differ
diff --git a/apps/website/public/widgets/nn-relu/index.html b/apps/website/public/widgets/nn-relu/index.html
new file mode 100644
index 0000000000..78fd09b8fe
--- /dev/null
+++ b/apps/website/public/widgets/nn-relu/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+ReLU: what a neuron does with a negative number · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
ReLU: what a neuron does with a negative number
+
Drag along the curve and watch ReLU pass every positive input and zero every negative one; give it a leaky slope and a little of the negative side gets through.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-relu/logic.js b/apps/website/public/widgets/nn-relu/logic.js
new file mode 100644
index 0000000000..2c5e16b7b8
--- /dev/null
+++ b/apps/website/public/widgets/nn-relu/logic.js
@@ -0,0 +1,32 @@
+// Generated by scripts/t27-logic.mjs from relu_activation.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 0b516ac7e0d25ec67b948833098b21421a1c385cb51abc49146e5855f849f559
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-relu/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABDgJgAAF9YAV9fX19fwF9AwQDAAEBBAUBcAEBAQUDAQARBgkBfwFBgIDAAAsHPwQGbWVtb3J5AgAPdDI3X3ZhbGlkX2lucHV0AAAPdDI3X2JhY2t3YXJkX2F0AAEOdDI3X2ZvcndhcmRfYXQAAgrXAgMHAEMAAADAC6QBAQJ/I4CAgIAAQSBrIgUkgICAgABBACEGIAVBGGpBACgCiIDAgAA2AgAgBSACOAIMIAUgATgCCCAFIAA4AgQgBUEAKQKAgMCAADcDEAJAA0AgBkEMRg0BIAVBEGogBmpDAACAPyADIAVBBGogBmoqAgBDAAAAAF4bOAIAIAZBBGohBgwACwsgBUEQaiAEQQJ0aioCACEDIAVBIGokgICAgAAgAwumAQECfyOAgICAAEEgayIFJICAgIAAQQAhBiAFQRhqQQAoAoiAwIAANgIAIAUgAjgCDCAFIAE4AgggBSAAOAIEIAVBACkCgIDAgAA3AxACQANAIAZBDEYNASAFQRBqIAZqIAVBBGogBmoqAgAiAiADIAKUIAJDAAAAAF4bOAIAIAZBBGohBgwACwsgBUEQaiAEQQJ0aioCACECIAVBIGokgICAgAAgAgsLFQEAQYCAwAALDAAAEEEAABBBAAAQQQ==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function forwardAt(a, b, c, slope, index) {
+ return W.t27_forward_at(a, b, c, slope, int(index, 0, 4294967295, "forward_at.index"));
+}
+function backwardAt(a, b, c, slope, index) {
+ return W.t27_backward_at(a, b, c, slope, int(index, 0, 4294967295, "backward_at.index"));
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ backwardAt,
+ forwardAt,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-relu/logic.t27 b/apps/website/public/widgets/nn-relu/logic.t27
new file mode 100644
index 0000000000..b726401b32
--- /dev/null
+++ b/apps/website/public/widgets/nn-relu/logic.t27
@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// ReLU produces sparse gradients | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: the tests called forward and backward with one or two of their three or
+// four arguments, two asserted only `result != undefined`, and the invariants
+// asserted `true`. Each test now runs the functions on buffers and checks values.
+
+module Relu;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const ZERO : f32 = 0.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const ReLUConfig = struct {
+ negative_slope : f32,
+ inplace : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(input: []const f32, output: []f32, config: ReLUConfig) → void
+ fn forward(input: []const f32, output: []f32, config: ReLUConfig) -> void {
+ // Values at or below ZERO take the negative branch, values above ZERO pass through
+ // For negative_slope = 0.0, this gives max(0, x) behavior
+ var i : usize = 0;
+ while (i < input.len) : (i += 1) {
+ if (input[i] > ZERO) {
+ output[i] = input[i]; // Positive branch: identity
+ } else {
+ output[i] = input[i] * config.negative_slope; // Negative branch: scaled
+ }
+ }
+ }
+
+ // backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) → void
+ fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) -> void {
+ // The gradient gate is on the forward input, not grad_output - this makes gradients sparse
+ var i : usize = 0;
+ while (i < input.len) : (i += 1) {
+ if (input[i] > ZERO) {
+ grad_input[i] = grad_output[i]; // Positive branch: identity gradient
+ } else {
+ grad_input[i] = grad_output[i] * config.negative_slope; // Negative branch: scaled gradient
+ }
+ }
+ }
+
+ // Note: config.inplace is declared but currently unused by these functions.
+ // The buffers are separate parameters, so the aliasing case is not handled.
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ fn cfg(slope: f32) -> ReLUConfig {
+ return ReLUConfig{ .negative_slope = slope, .inplace = false };
+ }
+
+ // output[index] of forward over (a, b, c)
+ fn forward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const input : [3]f32 = [a, b, c];
+ var output : [3]f32 = [9.0] ** 3;
+ forward(&input, &output, cfg(slope));
+ return output[index];
+ }
+
+ // grad_input[index] of backward over (a, b, c), every grad_output 1
+ fn backward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [1.0, 1.0, 1.0];
+ const input : [3]f32 = [a, b, c];
+ var grad_input : [3]f32 = [9.0] ** 3;
+ backward(&grad_output, &input, &grad_input, cfg(slope));
+ return grad_input[index];
+ }
+
+ // The shared fixture: one negative, zero, one positive.
+ fn valid_input() -> f32 {
+ return -2.0;
+ }
+
+ // max(0, x) on (-2, 0, 2)
+ test "forward_basic_case"
+ given negative = forward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = forward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and forward_at(-2.0, 0.0, 2.0, 0.0, 1) == 0.0 and positive == 2.0
+
+ test "forward_positive_identity"
+ given slope = 0.0
+ when first = forward_at(1.0, 2.0, 3.0, slope, 0)
+ then first == 1.0 and forward_at(1.0, 2.0, 3.0, slope, 1) == 2.0 and forward_at(1.0, 2.0, 3.0, slope, 2) == 3.0
+
+ test "forward_negative_scaling"
+ given slope = 0.5
+ when first = forward_at(-1.0, -2.0, -3.0, slope, 0)
+ then first == -0.5 and forward_at(-1.0, -2.0, -3.0, slope, 1) == -1.0 and forward_at(-1.0, -2.0, -3.0, slope, 2) == -1.5
+
+ test "forward_zero_negative_slope"
+ given slope = 0.0
+ when first = forward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 1) == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+
+ // grad_output * negative_slope for input <= ZERO (0 included), grad_output above it
+ test "backward_input_gate"
+ given slope = 0.5
+ when first = backward_at(-1.0, 0.0, 1.0, slope, 0)
+ then first == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 1) == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0
+
+ // Plain ReLU: the gradient is 0 left of zero and passes through right of it
+ test "backward_basic_case"
+ given negative = backward_at(-2.0, 0.0, 2.0, 0.0, 0)
+ when positive = backward_at(-2.0, 0.0, 2.0, 0.0, 2)
+ then negative == 0.0 and positive == 1.0
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input.len == output.len: forward writes every element of its buffer (none keeps the 9.0 fill)
+ invariant relu_constraint_0
+ assert forward_at(valid_input(), 0.0, 1.0, 0.0, 0) != 9.0 and forward_at(valid_input(), 0.0, 1.0, 0.0, 2) != 9.0
+
+ // A finite input gives a finite output: ReLU never exceeds its input's magnitude
+ invariant relu_constraint_1
+ assert @abs(forward_at(valid_input(), 0.0, 1.0, 0.5, 0)) <= @abs(valid_input())
diff --git a/apps/website/public/widgets/nn-relu/nn-relu.t27 b/apps/website/public/widgets/nn-relu/nn-relu.t27
new file mode 100644
index 0000000000..b63bcb47e7
--- /dev/null
+++ b/apps/website/public/widgets/nn-relu/nn-relu.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-relu.t27 -- ReLU and leaky ReLU: what a neuron does with a negative number
+; Source of truth for public/widgets/nn-relu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-relu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/relu_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 1 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -4 to 4 and sets a leaky slope from 0 to 0.3. The green
+; curve is forward_at over x and the dashed curve backward_at, the slope; the tiles read both at x
+; and over the spec's own three-input batch (-2, 0, 2).
+
+; WHAT IT DOES NOT CLAIM: ReLU is one choice among the activations of module 2; the page measures
+; nothing about training.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_relu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-relu";
+pub const TITLE : str = "ReLU: what a neuron does with a negative number";
+pub const DESCRIPTION : str = "Drag along the curve and watch ReLU pass every positive input and zero every negative one; give it a leaky slope and a little of the negative side gets through.";
+pub const IMAGE_ALT : str = "ReLU card: leaky slope 0.1 at x = -2. The green curve is flat at -0.2 on the left and rises at 45 degrees on the right; the dashed gold slope sits at 0.1 then jumps to 1. Tiles read out -0.200, slope 0.100 and the spec's batch (-2, 0, 2) as -0.20 0.00 2.00.";
+pub const HOOK : str = "max(0, x) is the whole rule, and its slope is 1 or 0. A leaky slope of 0.1 keeps a gradient alive on the negative side.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/relu_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 1 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -400;
+pub const K_X_HI_C : i16 = 400;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_Y_LO_C : i16 = -100;
+pub const K_Y_HI_C : i16 = 400;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [6]i8 = [-1, 0, 1, 2, 3, 4];
+pub const K_SLOPE_MAX_C : u8 = 30;
+pub const K_FIXTURE_C : [3]i16 = [-200, 0, 200];
+pub const K_PRESET_X_C : [4]i16 = [-200, 150, -200, -300];
+pub const K_PRESET_SLOPE_C : [4]u8 = [10, 0, 0, 30];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Leaky 0.1 at x = -2", "ReLU at x = 1.5", "ReLU at x = -2", "Leaky 0.3 at x = -3"];
+pub const SAY_ASK : str = "What does a neuron do with a negative number?";
+pub const SAY_X : str = "input x";
+pub const SAY_SLOPE : str = "leaky slope";
+pub const SAY_F : str = "ReLU(x)";
+pub const SAY_DF : str = "slope dReLU/dx";
+pub const SAY_PLOT_LABEL : str = "ReLU of x and its slope from -4 to 4, with a cursor you drag";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_BATCH : str = "spec batch -2 0 2";
+pub const SAY_BATCH_GRAD : str = "its slopes";
+pub const SAY_NOTE : str = "A slope of 0 means no gradient flows back, so a unit that only ever sees negative inputs stops learning. The leaky slope is the fix the spec takes as its fourth argument.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/relu_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_range_holds_both_sides {
+ assert K_X_LO_C < 0;
+ assert K_X_HI_C > 0;
+ assert K_X_LO_C + K_X_HI_C == 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_X_C[3] >= K_X_LO_C;
+ assert K_PRESET_SLOPE_C[3] <= K_SLOPE_MAX_C;
+ assert K_PRESET_SLOPE_C[1] == 0;
+ assert K_FIXTURE_C[0] == K_PRESET_X_C[0];
+}
diff --git a/apps/website/public/widgets/nn-relu/tool.js b/apps/website/public/widgets/nn-relu/tool.js
new file mode 100644
index 0000000000..178ed0cf13
--- /dev/null
+++ b/apps/website/public/widgets/nn-relu/tool.js
@@ -0,0 +1,38 @@
+// nn-relu/tool.js -- lesson 1 of neural-networks: what a neuron does with a negative number. ReLU
+// passes a positive input and zeroes a negative one; a leaky slope lets a little through. It computes
+// nothing itself: every point of both curves and every tile is forward_at and backward_at of
+// gHashTag/t27 specs/ml/activation/relu_activation.t27, compiled to wasm (./logic.js). Words are SAY_
+// and numbers K_ constants of specs/widgets/nn-relu.t27.
+import { backwardAt, forwardAt } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const X0 = centi(K('K_X_LO_C', -400)), X1 = centi(K('K_X_HI_C', 400))
+const f = (x, s) => forwardAt(x, 0, 0, s, 0)
+const df = (x, s) => backwardAt(x, 0, 0, s, 0)
+const ctl = controls([
+ { key: 'x', label: say('SAY_X'), min: X0, max: X1, step: centi(K('K_X_STEP_C', 5)), fmt: (v) => fmt(v, 2) },
+ { key: 'slope', label: say('SAY_SLOPE'), min: 0, max: centi(K('K_SLOPE_MAX_C', 30)), step: 0.01, fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { x: K('K_PRESET_X_C', []).map(centi), slope: K('K_PRESET_SLOPE_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_F'), COLORS[0]], [say('SAY_DF'), COLORS[1], true]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 340, x0: X0, x1: X1, y0: centi(K('K_Y_LO_C', -100)), y1: centi(K('K_Y_HI_C', 400)), xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_X') }
+ const p = plot(svg, o)
+ p.line(sample((x) => f(x, s.slope), X0, X1), COLORS[0], 3)
+ p.line(sample((x) => df(x, s.slope), X0, X1, 320), COLORS[1], 2, { 'stroke-dasharray': '6 5' })
+ cursor(svg, p, s.x, [[f(s.x, s.slope), COLORS[0]], [df(s.x, s.slope), COLORS[1]]])
+ dragX(svg, p, o, (x) => ctl.set('x', x))
+ const fx = K('K_FIXTURE_C', []).map(centi)
+ stat.replaceChildren(tiles([
+ { label: say('SAY_OUT'), value: fmt(f(s.x, s.slope), 3), tone: 0 },
+ { label: say('SAY_GRAD'), value: fmt(df(s.x, s.slope), 3), tone: 1 },
+ { label: say('SAY_BATCH'), value: [0, 1, 2].map((i) => fmt(forwardAt(fx[0], fx[1], fx[2], s.slope, i), 2)).join(' '), tone: 2 },
+ { label: say('SAY_BATCH_GRAD'), value: [0, 1, 2].map((i) => fmt(backwardAt(fx[0], fx[1], fx[2], s.slope, i), 2)).join(' '), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-rmsprop/card.png b/apps/website/public/widgets/nn-rmsprop/card.png
new file mode 100644
index 0000000000..d198e00215
Binary files /dev/null and b/apps/website/public/widgets/nn-rmsprop/card.png differ
diff --git a/apps/website/public/widgets/nn-rmsprop/index.html b/apps/website/public/widgets/nn-rmsprop/index.html
new file mode 100644
index 0000000000..cbcb1194e5
--- /dev/null
+++ b/apps/website/public/widgets/nn-rmsprop/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+RMSProp: a step that does not care how big the gradient is · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
RMSProp: a step that does not care how big the gradient is
+
Feed one weight a constant gradient of 0.001, 0.5 or 1000: RMSProp divides by the root of its running mean square, so all three walk the same path. Change the decay and the path bends.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-rmsprop/logic.js b/apps/website/public/widgets/nn-rmsprop/logic.js
new file mode 100644
index 0000000000..6b4d240fb5
--- /dev/null
+++ b/apps/website/public/widgets/nn-rmsprop/logic.js
@@ -0,0 +1,28 @@
+// Generated by scripts/t27-logic.mjs from rmsprop.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: f483ccb6c0ffa9f2b941a64d00daae8d69823fb982fd2f2b1a9fca9638c44ab7
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-rmsprop/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABDgJgAn1/AX1gA31/fQF9AwMCAAEEBQFwAQEBBQMBABAGCQF/AUGAgMAACwcuAwZtZW1vcnkCAA90MjdfY2FjaGVfYWZ0ZXIAAA90MjdfcGFyYW1fYWZ0ZXIAAQqZAQI7AQF9IAAgAEPQzMw9lJQhAkMAAAAAIQACQANAIAFFDQEgAUF/aiEBIAIgAENmZmY/lJIhAAwACwsgAAtbAQN9IABDCtcjPJQhA0MAAIA/IQQgACAAQwAAgD8gApOUlCEFQwAAAAAhAAJAA0AgAUUNASABQX9qIQEgBCADIAUgAiAAlJIiAJFDd8wrMpKVkyEEDAALCyAECw==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function paramAfter(g, steps, decay) {
+ return W.t27_param_after(g, int(steps, 0, 4294967295, "param_after.steps"), decay);
+}
+function cacheAfter(g, steps) {
+ return W.t27_cache_after(g, int(steps, 0, 4294967295, "cache_after.steps"));
+}
+const __NOT_LOWERED__ = Object.freeze(["step"]);
+export {
+ __NOT_LOWERED__,
+ cacheAfter,
+ paramAfter
+};
diff --git a/apps/website/public/widgets/nn-rmsprop/logic.t27 b/apps/website/public/widgets/nn-rmsprop/logic.t27
new file mode 100644
index 0000000000..ab0e105933
--- /dev/null
+++ b/apps/website/public/widgets/nn-rmsprop/logic.t27
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// RMSprop optimizer with moving average of squared gradients | φ² + 1/φ² = 3 | TRINITY
+// Restored 2026-10-05 from trinity specs/algo/rmsprop.tri. The port kept only
+// `grad`, squared grad[0], and threw it away; the test read an undefined
+// default_input().
+
+module Rmsprop;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_DECAY : f32 = 0.9;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const RMSpropConfig = struct {
+ learning_rate : f32,
+ decay : f32,
+ epsilon : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // step(grad, param, cache, config) -> void, in place:
+ // cache = decay * cache + (1 - decay) * grad^2
+ // param -= lr * grad / (sqrt(cache) + epsilon)
+ // The .tri formula reads lr, decay and epsilon but its parameter list
+ // omits them; they come from config here.
+ fn step(grad: []const f32, param: []f32, cache: []f32, config: RMSpropConfig) -> void {
+ var i : usize = 0;
+ while (i < grad.len) {
+ const g = grad[i];
+ cache[i] = config.decay * cache[i] + (1.0 - config.decay) * g * g;
+ param[i] = param[i] - config.learning_rate * g / (@sqrt(cache[i]) + config.epsilon);
+ i = i + 1;
+ }
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests
+ // ═══════════════════════════════════════════════════════════
+
+ // run `steps` steps of constant gradient g on one parameter from 1.0
+ fn param_after(g: f32, steps: usize, decay: f32) -> f32 {
+ const grad : [1]f32 = [g];
+ var param : [1]f32 = [1.0];
+ var cache : [1]f32 = [0.0];
+ var k : usize = 0;
+ while (k < steps) {
+ step(&grad, ¶m, &cache, RMSpropConfig{ .learning_rate = 0.01, .decay = decay, .epsilon = 0.00000001 });
+ k = k + 1;
+ }
+ return param[0];
+ }
+
+ fn cache_after(g: f32, steps: usize) -> f32 {
+ const grad : [1]f32 = [g];
+ var param : [1]f32 = [1.0];
+ var cache : [1]f32 = [0.0];
+ var k : usize = 0;
+ while (k < steps) {
+ step(&grad, ¶m, &cache, RMSpropConfig{ .learning_rate = 0.01, .decay = DEFAULT_DECAY, .epsilon = 0.00000001 });
+ k = k + 1;
+ }
+ return cache[0];
+ }
+
+ // first step from cache 0: cache = 0.1 g^2, so the move is lr / sqrt(0.1)
+ // = 0.0316228 whatever the size of g
+ test first_step_is_scale_free
+ given small = param_after(0.001, 1, DEFAULT_DECAY)
+ when large = param_after(1000.0, 1, DEFAULT_DECAY)
+ then @abs(small - (1.0 - 0.0316228)) < 0.00001 and @abs(large - small) < 0.00001
+
+ // moving average of a constant: cache_n = (1 - 0.9^n) g^2
+ test cache_is_moving_average
+ given result = cache_after(2.0, 3)
+ when expected = (1.0 - 0.729) * 4.0
+ then @abs(result - expected) < 0.00001
+
+ test moves_against_the_gradient
+ given up = param_after(0.5, 2, DEFAULT_DECAY)
+ when down = param_after(-0.5, 2, DEFAULT_DECAY)
+ then up < 1.0 and down > 1.0 and @abs((1.0 - up) - (down - 1.0)) < 0.00001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants
+ // ═══════════════════════════════════════════════════════════
+
+ invariant rmsprop_decay_in_unit_interval
+ assert DEFAULT_DECAY > 0.0 and DEFAULT_DECAY < 1.0
diff --git a/apps/website/public/widgets/nn-rmsprop/nn-rmsprop.t27 b/apps/website/public/widgets/nn-rmsprop/nn-rmsprop.t27
new file mode 100644
index 0000000000..1ee610febf
--- /dev/null
+++ b/apps/website/public/widgets/nn-rmsprop/nn-rmsprop.t27
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-rmsprop.t27 -- RMSProp: a step that ignores the size of the gradient
+; Source of truth for public/widgets/nn-rmsprop/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-rmsprop/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/rmsprop.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 17 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader picks a gradient, a decay from 0.5 to 0.99 and a number of steps. Every
+; point is param_after(g, n, decay) from 1.0 with lr 0.01, the cache cache_after(g, n) at the spec's
+; decay 0.9.
+
+; WHAT IT DOES NOT CLAIM: a constant gradient on one weight; with a changing gradient the cache lags
+; behind it.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_rmsprop;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-rmsprop";
+pub const TITLE : str = "RMSProp: a step that does not care how big the gradient is";
+pub const DESCRIPTION : str = "Feed one weight a constant gradient of 0.001, 0.5 or 1000: RMSProp divides by the root of its running mean square, so all three walk the same path. Change the decay and the path bends.";
+pub const IMAGE_ALT : str = "RMSProp card: gradient 0.5, decay 0.9, 100 steps. Three lines, green for 0.5 and dashed gold and pink for 0.001 and 1000, lie exactly on top of each other, falling from 1 to about -0.09. Tiles read the weight -0.09142, the three runs -0.0914 -0.0914 -0.0914, one step 0.96838 and the cache 0.249993.";
+pub const HOOK : str = "Its first step is 0.0316 whether the gradient is 0.001 or 1000. Dividing by the gradient's own size is what makes the step scale-free.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/rmsprop.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 17 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS_HI : u8 = 100;
+pub const K_GRADS_M : [3]u32 = [1, 500, 1000000];
+pub const K_DECAY_LO_C : u8 = 50;
+pub const K_DECAY_HI_C : u8 = 99;
+pub const K_Y_LO_C : i16 = -110;
+pub const K_Y_HI_C : u8 = 110;
+pub const K_Y_TICKS_C : [5]i8 = [-100, -50, 0, 50, 100];
+pub const K_STEP_TICKS : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_G : [4]u8 = [1, 0, 2, 1];
+pub const K_PRESET_DECAY_C : [4]u8 = [90, 90, 90, 99];
+pub const K_PRESET_STEPS : [4]u8 = [100, 100, 100, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["g = 0.5", "g = 0.001", "g = 1000", "g = 0.5, decay 0.99"];
+pub const SAY_ASK : str = "Can a step ignore how big the gradient is?";
+pub const SAY_GRAD : str = "gradient";
+pub const SAY_GRAD_NAMES : [3]str = ["g = 0.001", "g = 0.5", "g = 1000"];
+pub const SAY_DECAY : str = "decay";
+pub const SAY_STEPS : str = "steps";
+pub const SAY_PLOT_LABEL : str = "The weight after every step for three gradient sizes";
+pub const SAY_W_NOW : str = "weight now";
+pub const SAY_W_OTHERS : str = "all three gradients";
+pub const SAY_ONE_STEP : str = "after step 1";
+pub const SAY_CACHE : str = "cache, decay 0.9";
+pub const SAY_NOTE : str = "The cache is a running mean of g squared, so g / sqrt(cache) has no units: a layer whose gradients are tiny moves as fast as one whose gradients are huge.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/rmsprop.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_gradients_span_six_decades {
+ assert K_GRADS_M[0] == 1;
+ assert K_GRADS_M[2] == 1000000;
+ assert K_GRADS_M[1] == 500;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_G[2] < 3;
+ assert K_PRESET_DECAY_C[3] == K_DECAY_HI_C;
+ assert K_PRESET_STEPS[0] == K_STEPS_HI;
+}
diff --git a/apps/website/public/widgets/nn-rmsprop/tool.js b/apps/website/public/widgets/nn-rmsprop/tool.js
new file mode 100644
index 0000000000..6fb4803561
--- /dev/null
+++ b/apps/website/public/widgets/nn-rmsprop/tool.js
@@ -0,0 +1,37 @@
+// nn-rmsprop/tool.js -- lesson 17 of neural-networks: a step that does not care how big the gradient
+// is. RMSProp divides each step by the root of a running mean of squared gradients, so a gradient
+// of 0.001 and one of 1000 move the weight alike. Every point is param_after and every cache value
+// cache_after of gHashTag/t27 specs/ml/optimizer/rmsprop.t27, compiled to wasm (./logic.js): a
+// constant gradient on one weight from 1.0, lr 0.01. Words are SAY_ and numbers K_ constants of
+// specs/widgets/nn-rmsprop.t27.
+import { cacheAfter, paramAfter } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, say, source, tiles } from '../nn-kit.js'
+
+const NMAX = K('K_STEPS_HI', 100)
+const G = K('K_GRADS_M', []).map((g) => g / 1000)
+const ctl = controls([
+ { key: 'g', label: say('SAY_GRAD'), kind: 'choice', options: W.SAY_GRAD_NAMES ?? [] },
+ { key: 'decay', label: say('SAY_DECAY'), min: centi(K('K_DECAY_LO_C', 50)), max: centi(K('K_DECAY_HI_C', 99)), step: 0.01, fmt: (v) => fmt(v, 2) },
+ { key: 'n', label: say('SAY_STEPS'), min: 1, max: NMAX },
+], { names: W.SAY_PRESET_NAMES, values: { g: K('K_PRESET_G', []), decay: K('K_PRESET_DECAY_C', []).map(centi), n: K('K_PRESET_STEPS', []) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend((W.SAY_GRAD_NAMES ?? []).map((w, i) => [w, COLORS[[0, 1, 3][i]], i > 0])), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 330, x0: 0, x1: NMAX, y0: centi(K('K_Y_LO_C', -10)), y1: centi(K('K_Y_HI_C', 110)), xticks: K('K_STEP_TICKS', []), yticks: K('K_Y_TICKS_C', []).map(centi), xlabel: say('SAY_STEPS') }
+ const p = plot(svg, o)
+ G.forEach((g, i) => p.line([...Array(NMAX + 1)].map((_, k) => [k, paramAfter(g, k, s.decay)]), COLORS[[0, 1, 3][i]], i === s.g ? 5 : 2, i === s.g ? {} : { 'stroke-dasharray': '6 6' }))
+ cursor(svg, p, s.n, [[paramAfter(G[s.g], s.n, s.decay), COLORS[[0, 1, 3][s.g]]]])
+ dragX(svg, p, o, (n) => ctl.set('n', Math.round(n)))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_W_NOW'), value: fmt(paramAfter(G[s.g], s.n, s.decay), 5), tone: 0 },
+ { label: say('SAY_W_OTHERS'), value: G.map((g) => fmt(paramAfter(g, s.n, s.decay), 4)).join(' '), tone: 1 },
+ { label: say('SAY_ONE_STEP'), value: fmt(paramAfter(G[s.g], 1, s.decay), 5), tone: 2 },
+ { label: say('SAY_CACHE'), value: fmt(cacheAfter(G[s.g], s.n), 6), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-rnn/card.png b/apps/website/public/widgets/nn-rnn/card.png
new file mode 100644
index 0000000000..b61326514d
Binary files /dev/null and b/apps/website/public/widgets/nn-rnn/card.png differ
diff --git a/apps/website/public/widgets/nn-rnn/index.html b/apps/website/public/widgets/nn-rnn/index.html
new file mode 100644
index 0000000000..7260d8bcf0
--- /dev/null
+++ b/apps/website/public/widgets/nn-rnn/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Recurrent cells: how a network remembers a moment ago · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Recurrent cells: how a network remembers a moment ago
+
Tap the inputs of eight steps: the cell feeds its own last output back, h' = tanh(x + 0.5 h), so every pulse echoes on and fades by about half a step. Two pulses add up.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-rnn/logic.js b/apps/website/public/widgets/nn-rnn/logic.js
new file mode 100644
index 0000000000..98f888ea96
--- /dev/null
+++ b/apps/website/public/widgets/nn-rnn/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from rnn_cell.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 09f67415a49b61d028c7a15e93caff8f3d3d309b7bb9452b1b6260132caf8887
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-rnn/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABIgZgAAF9YAJ9fQF9YAV/f39/fwBgAX0BfWABfwF/YAF/AX0DCQgAAQIDBAUFAQQFAXABAQEFAwEAEQYJAX8BQYCAwAALB18GBm1lbW9yeQIAD3QyN192YWxpZF9pbnB1dAAADHQyN19pbml0X2xlbgAEDXQyN193aWRlX3N0ZXAABRF0MjdfdGlueV9zZXF1ZW5jZQAGDXQyN190aW55X3N0ZXAABwqSEAgSAEMAAIA/QwAAAAAQgYCAgAALvgEBAX8jgICAgABBwABrIgIkgICAgAAgAkGAgID4AzYCDCACQYCAgPwDNgIIIAJBADYCECACIAA4AhQgAiABOAIYIAJBgIDAiAQ2AhwgAkEBNgI0IAJBATYCLCACQQE2AiQgAkKBgICAEDcDOCACIAJBEGo2AjAgAiACQQxqNgIoIAIgAkEIajYCICACQRRqIAJBGGogAkEgaiACQThqIAJBHGoQgoCAgAAgAioCHCEAIAJBwABqJICAgIAAIAAL2gMECn8BfQF/AX0jgICAgABBEGsiBSSAgICAACADKAIEIgZBAnQhByADKAIAIghBAnQhCSACKAIQIQogAigCCCELIAIoAgAhDEEAIQ0DQAJAAkACQCANIAZGDQAgCiANQQJ0Ig5qKgIAIQ8gACEDIAwhECAIIQIDQAJAIAINACABIQMgCyEQIAYhAgJAA0AgAkUNASACQX9qIQIgDyAQKgIAIAMqAgCUkiEPIANBBGohAyAQQQRqIRAMAAsLIAQgDmohAwJAIA+LIhG8IgJB1b6y+ANJDQACQCACQYCAgIkETQ0AQwAAAAAgD5VDAACAP5IhEQwGC0MAAIA/QwAAAEAgESARkhCDgICAAEMAAABAkpWTIREMBQsgAkH4iov0A0sNAwJAIAJBgICABEkNACARQwAAAMCUEIOAgIAAIhGMIBFDAAAAQJKVIREMBQsgBSAPIA+UOAIMIAVBDGohAgwECyACQX9qIQIgDyAQKgIAIAMqAgCUkiEPIANBBGohAyAQQQRqIRAMAAsLIAVBEGokgICAgAAPCyARIBGSEIOAgIAAIhEgEUMAAABAkpUhEQsgAyARjCARIA+8QQBIGzgCACALIAdqIQsgDCAJaiEMIA1BAWohDQwACwu4BQQBfwF9An8DfSOAgICAAEEQayIBJICAgIAAAkACQCAAIABbDQBDAADAfyECDAELQwAAgL8hAiAAQwAAgP9bDQACQAJAAkACQAJAAkACQAJAIAC8IgNB/////wdxIgRBw/DWjARLDQAgBEGY5MX1A00NASAEQZKrlPwDTw0CQX8hBCADQX9MDQMgAEOAcTG/kiECQ9H3FzchBUEBIQQMBgsgBEGAgID8B0sNBCADQQBIDQcgAEOAcbFCXkUNASAAQwAAAH+UIQIMBwsgBEGAgICYA0kNAkMAAAAAIQVBACEEDAULIABDO6q4P5RDAAAAPyAAmJL8ACIEsiICQ9H3FzeUIQUgACACQ4BxMb+UkiECDAMLIABDgHExP5IhAkPR9xe3IQUMAgsgBEH///8DSw0AIAEgACAAlDgCDCABQQxqIQQLIAAhAgwCCyACIAIgBZMiAJMgBZMhBQsgACAAQwAAAD+UIgaUIgIgAiACQxAwzzqUQ2iICL2SlEMAAIA/kiIHQwAAQEAgBiAHlJMiBpNDAADAQCAAIAaUk5WUIQYCQCAEDQAgACAAIAaUIAKTkyECDAELIAAgBiAFk5QgBZMgApMhAgJAAkACQAJAIARBAWoOAwACAQILIAAgApNDAAAAP5RDAAAAv5IhAgwDCyAAQwAAgL5dDQEgACACkyIAIACSQwAAgD+SIQIMAgsgBEEXdCIDQYCAgPwDar4hBQJAIARBOEsNAEGAgID8AyADa74hBgJAIARBFksNAEMAAIA/IAaTIAAgApOSIAWUIQIMAwsgACACIAaSk0MAAIA/kiAFlCECDAILIAAgApNDAACAP5IiACAAkkMAAAB/lCAAIAWUIARBgAFGG0MAAIC/kiECDAELIAIgAEMAAAA/kpNDAAAAwJQhAgsgAUEQaiSAgICAACACC4MBAgN/AX0jgICAgABBgAJrIgEkgICAgAACQEGAAkUNACABQYCAwIAAQYAC/AoAAAsgASECIAAhAwJAA0AgA0UNASACQQA2AgAgA0F/aiEDIAJBBGohAgwACwsgASAAQQJ0akF8aioCACEEIAFBgAJqJICAgIAAQQAgACAEQwAAAABcGwvkAQIBfwF9I4CAgIAAQdAAayIBJICAgIAAIAFBBDYCRCABQQQ2AjwgAUKAgMCIhICAiMEANwMwIAFCgICAgICAgIC/fzcCKCABQoCAgIDQmbPmPTcCICABQs2Zs+4DNwIYIAFCgICA+IOAgIA/NwIQIAFCgICA/IOAgMC/fzcCCCABIAFBKGo2AkggASABQRhqNgJAIAEgAUEIajYCOCABQQI2AkxBgILAgABBiILAgAAgAUE4akGQgsCAACABQTBqEIKAgIAAIAFBMGogAEECdGoqAgAhAiABQdAAaiSAgICAACACC64CAgR/AX0jgICAgABBwABrIgEkgICAgABBACECIAFBGGpBACgCoILAgAA2AgAgAUKBgICAEDcDOCABQQE2AjQgAUEBNgIsIAFBATYCJCABQQApApiCwIAANwMQIAFBADYCDCABQYCAgPgDNgIIIAFBgICA/AM2AgQgASABQQxqNgIwIAEgAUEIajYCKCABIAFBBGo2AiBBACEDA30CQAJAAkACQCADDgQAAQECAQtBpILAgABBsILAgAAgAUEgaiABQThqIAFBEGoQgoCAgAAMAgsgAkGkgsCAAGogAUEQaiACaiIEQXxqIAFBIGogAUE4aiAEEIKAgIAADAELIAFBEGogAEECdGoqAgAhBSABQcAAaiSAgICAACAFDwsgAkEEaiECIANBAWohAwwACwsMACAAIAEQgYCAgAALC74CAQBBgIDAAAu0AgAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAAIA/AAAAQAAAgD8AAIC/AgAAAAIAAAAAABBBAAAQQQAAEEEAAIA/AAAAAAAAAAAAAAAA";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function tinyStep(x, h) {
+ return W.t27_tiny_step(x, h);
+}
+function tinySequence(t) {
+ return W.t27_tiny_sequence(int(t, 0, 4294967295, "tiny_sequence.t"));
+}
+function wideStep(index) {
+ return W.t27_wide_step(int(index, 0, 4294967295, "wide_step.index"));
+}
+function initLen(hiddenSize) {
+ return W.t27_init_len(int(hiddenSize, 0, 4294967295, "init_len.hidden_size")) >>> 0;
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+const __NOT_LOWERED__ = Object.freeze(["forward_step", "forward_sequence", "init_state", "tiny"]);
+export {
+ __NOT_LOWERED__,
+ initLen,
+ tinySequence,
+ tinyStep,
+ validInput,
+ wideStep
+};
diff --git a/apps/website/public/widgets/nn-rnn/logic.t27 b/apps/website/public/widgets/nn-rnn/logic.t27
new file mode 100644
index 0000000000..0ad0f92fae
--- /dev/null
+++ b/apps/website/public/widgets/nn-rnn/logic.t27
@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Basic RNN recurrence equation | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written against undeclared `matmul`, `add`, `zeros`, `len` and `append`,
+// it applied tanh twice (h = tanh(tanh(z))), and the tests asserted
+// `result != undefined`. It is now the Elman cell,
+// h' = tanh(W_x x + W_h h + b), with W_x hidden x input and W_h hidden x hidden
+// (row-major), stepping over buffers the caller owns.
+
+module RnnCell;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_INPUT_SIZE : u32 = 128;
+ const DEFAULT_HIDDEN_SIZE : u32 = 128;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const RNNCellConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ };
+
+ pub const RNNState = struct {
+ hidden : []f32,
+ cell : []f32,
+ };
+
+ pub const RNNParams = struct {
+ w_x : []f32,
+ w_h : []f32,
+ b : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // new_hidden[i] = tanh(sum_j W_x[i][j] x[j] + sum_k W_h[i][k] h[k] + b[i])
+ fn forward_step(input: []const f32, hidden: []const f32, params: RNNParams, config: RNNCellConfig, new_hidden: []f32) -> void {
+ var i : usize = 0;
+ while (i < config.hidden_size) {
+ var z : f32 = params.b[i];
+ var j : usize = 0;
+ while (j < config.input_size) {
+ z = z + params.w_x[i * config.input_size + j] * input[j];
+ j = j + 1;
+ }
+ var k : usize = 0;
+ while (k < config.hidden_size) {
+ z = z + params.w_h[i * config.hidden_size + k] * hidden[k];
+ k = k + 1;
+ }
+ new_hidden[i] = tanh(z);
+ i = i + 1;
+ }
+ }
+
+ // inputs: steps x input_size; outputs: steps x hidden_size, row t = h after step t.
+ // The state starts at initial_state and each step reads the previous row.
+ fn forward_sequence(inputs: []const f32, steps: usize, initial_state: []const f32, params: RNNParams, config: RNNCellConfig, outputs: []f32) -> void {
+ const n_in : usize = config.input_size;
+ const n_h : usize = config.hidden_size;
+ var t : usize = 0;
+ while (t < steps) {
+ if (t == 0) {
+ forward_step(inputs[0..n_in], initial_state, params, config, outputs[0..n_h]);
+ } else {
+ forward_step(inputs[t * n_in .. (t + 1) * n_in], outputs[(t - 1) * n_h .. t * n_h], params, config, outputs[t * n_h .. (t + 1) * n_h]);
+ }
+ t = t + 1;
+ }
+ }
+
+ // The first hidden_size entries of `buffer`, zeroed
+ fn init_state(buffer: []f32, hidden_size: u32) -> []f32 {
+ var i : usize = 0;
+ while (i < hidden_size) {
+ buffer[i] = 0.0;
+ i = i + 1;
+ }
+ return buffer[0..hidden_size];
+ }
+
+ // A 1-in, 1-hidden cell with W_x = 1, W_h = 0.5, b = 0: h' = tanh(x + 0.5 h)
+ fn tiny() -> RNNCellConfig {
+ return RNNCellConfig{ .input_size = 1, .hidden_size = 1 };
+ }
+
+ fn tiny_step(x: f32, h: f32) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const input : [1]f32 = [x];
+ const hidden : [1]f32 = [h];
+ var out : [1]f32 = [9.0];
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &out);
+ return out[0];
+ }
+
+ // The sequence (1, 0, 0) from h = 0: h after step t
+ fn tiny_sequence(t: usize) -> f32 {
+ var w_x : [1]f32 = [1.0];
+ var w_h : [1]f32 = [0.5];
+ var b : [1]f32 = [0.0];
+ const inputs : [3]f32 = [1.0, 0.0, 0.0];
+ const h0 : [1]f32 = [0.0];
+ var outputs : [3]f32 = [9.0] ** 3;
+ forward_sequence(&inputs, 3, &h0, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &outputs);
+ return outputs[t];
+ }
+
+ // A 2-in, 2-hidden cell: h' = tanh(W_x x + W_h h + b) with W_x = [[1, -1], [0.5, 0.5]],
+ // W_h = [[0.1, 0], [0, 0.1]], b = (0, -0.5), at x = (1, 2), h = (1, -1)
+ fn wide_step(index: usize) -> f32 {
+ var w_x : [4]f32 = [1.0, -1.0, 0.5, 0.5];
+ var w_h : [4]f32 = [0.1, 0.0, 0.0, 0.1];
+ var b : [2]f32 = [0.0, -0.5];
+ const input : [2]f32 = [1.0, 2.0];
+ const hidden : [2]f32 = [1.0, -1.0];
+ var out : [2]f32 = [9.0] ** 2;
+ forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, RNNCellConfig{ .input_size = 2, .hidden_size = 2 }, &out);
+ return out[index];
+ }
+
+ fn init_len(hidden_size: u32) -> usize {
+ var buffer : [64]f32 = [9.0] ** 64;
+ const h = init_state(&buffer, hidden_size);
+ if (h[hidden_size - 1] != 0.0) {
+ return 0;
+ }
+ return h.len;
+ }
+
+ fn valid_input() -> f32 {
+ return tiny_step(1.0, 0.0);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // z = (1 - 2 + 0.1, 0.5 + 1 - 0.1 - 0.5) = (-0.9, 0.9): h' = (tanh(-0.9), tanh(0.9))
+ test forward_step_basic_case
+ given h0 = wide_step(0)
+ when h1 = wide_step(1)
+ then @abs(h0 + 0.7162979) < 0.000001 and @abs(h1 - 0.7162979) < 0.000001 and @abs(tiny_step(1.0, 0.0) - 0.7615942) < 0.000001
+
+ // The cell remembers: after the input pulse, h decays through the recurrent weight,
+ // h1 = tanh(1), h2 = tanh(0.5 h1), h3 = tanh(0.5 h2)
+ test forward_sequence_basic_case
+ given h1 = tiny_sequence(0)
+ when h3 = tiny_sequence(2)
+ then @abs(h1 - 0.7615942) < 0.000001 and @abs(tiny_sequence(1) - tanh(0.5 * h1)) < 0.000001 and @abs(h3 - tanh(0.5 * tiny_sequence(1))) < 0.000001 and h3 > 0.0 and h3 < tiny_sequence(1)
+
+ test init_state_basic_case
+ given hidden_size = 64
+ when result = init_len(hidden_size)
+ then result == hidden_size
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input_size >= 1
+ invariant rnn_cell_constraint_0
+ assert DEFAULT_INPUT_SIZE >= 1
+
+ // hidden_size >= 1
+ invariant rnn_cell_constraint_1
+ assert DEFAULT_HIDDEN_SIZE >= 1
+
+ // name: rnn_formula -- one tanh, not two: tanh(1) = 0.7615942, tanh(tanh(1)) = 0.6420
+ invariant rnn_cell_constraint_2
+ assert valid_input() > 0.76 and valid_input() < 0.77
+
+ // "Elman (1990) - Finding Structure in Time": the state carries the past forward
+ invariant rnn_cell_constraint_3
+ assert tiny_step(0.0, 1.0) > 0.0
+
+ // "Goodfellow et al. (2016) - Sequence Modeling with RNNs and LSTMs": tanh bounds the state
+ invariant rnn_cell_constraint_4
+ assert tiny_step(100.0, 1.0) <= 1.0 and tiny_step(-100.0, -1.0) >= -1.0
diff --git a/apps/website/public/widgets/nn-rnn/nn-rnn.t27 b/apps/website/public/widgets/nn-rnn/nn-rnn.t27
new file mode 100644
index 0000000000..2335230222
--- /dev/null
+++ b/apps/website/public/widgets/nn-rnn/nn-rnn.t27
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-rnn.t27 -- a recurrent cell: an echo that fades
+; Source of truth for public/widgets/nn-rnn/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-rnn/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/recurrent/rnn_cell.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 22 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader taps each of eight inputs to cycle it through -1, 0 and +1. Each h is
+; tiny_step(x, h) of the spec, applied in order from h = 0; the spec's own three-step run is
+; tiny_sequence(t).
+
+; WHAT IT DOES NOT CLAIM: a 1-in, 1-hidden cell with the spec's weights (W_x 1, W_h 0.5, b 0); a
+; real cell has a vector of hidden units.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_rnn;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-rnn";
+pub const TITLE : str = "Recurrent cells: how a network remembers a moment ago";
+pub const DESCRIPTION : str = "Tap the inputs of eight steps: the cell feeds its own last output back, h' = tanh(x + 0.5 h), so every pulse echoes on and fades by about half a step. Two pulses add up.";
+pub const IMAGE_ALT : str = "RNN card: inputs +1, 0, 0, +1, 0, 0, 0, 0 as cyan bars, and the green hidden state jumping to 0.76, fading to 0.36 and 0.18, jumping to 0.80 at the second pulse and fading to 0.05. Tiles read the last h 0.0466 and the spec's own run 0.7616 0.3634 0.1797.";
+pub const HOOK : str = "A single pulse gives 0.762, 0.363, 0.180: the memory halves each step. A cell remembers, but not for long, which is the problem LSTMs solve.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/recurrent/rnn_cell.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 22 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS : u8 = 8;
+pub const K_Y_TICKS : [3]i8 = [-1, 0, 1];
+pub const K_PRESET_INPUTS : [32]i8 = [1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 1, -1, 1, -1, 1, -1, 1, 1, 1, 1, 1, 1, 1, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Two pulses", "One pulse", "Alternating", "Always on"];
+pub const SAY_ASK : str = "How does a network remember what it saw a moment ago?";
+pub const SAY_TAP : str = "Inputs: tap a step to change it";
+pub const SAY_STEP : str = "step";
+pub const SAY_X : str = "input x";
+pub const SAY_H : str = "hidden state h";
+pub const SAY_PLOT_LABEL : str = "Input and hidden state at every step";
+pub const SAY_H_LAST : str = "h, last step";
+pub const SAY_H_ALL : str = "h at every step";
+pub const SAY_SPEC_RUN : str = "spec run 1, 0, 0";
+pub const SAY_ECHO : str = "echo of one pulse";
+pub const SAY_NOTE : str = "The same weights act at every step, and the state h carries what came before. Because 0.5 h is squeezed by tanh each time, an old input fades geometrically.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/recurrent/rnn_cell.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test eight_steps_four_presets {
+ assert K_STEPS == 8;
+ assert K_PRESET_INPUTS[0] == 1;
+ assert K_PRESET_INPUTS[3] == 1;
+}
+
+test the_inputs_are_trits {
+ assert K_PRESET_INPUTS[17] + 1 == 0;
+ assert K_PRESET_INPUTS[31] == 1;
+ assert K_PRESET_INPUTS[8] == 1;
+ assert K_PRESET_INPUTS[9] == 0;
+}
diff --git a/apps/website/public/widgets/nn-rnn/tool.js b/apps/website/public/widgets/nn-rnn/tool.js
new file mode 100644
index 0000000000..7a7b152671
--- /dev/null
+++ b/apps/website/public/widgets/nn-rnn/tool.js
@@ -0,0 +1,44 @@
+// nn-rnn/tool.js -- lesson 22 of neural-networks: how a network remembers what it saw a moment ago.
+// A recurrent cell feeds its own last output back in: h' = tanh(x + 0.5 h), so a pulse echoes on and
+// fades by about half each step. Every h is tiny_step of gHashTag/t27 specs/ml/recurrent/
+// rnn_cell.t27, compiled to wasm (./logic.js), applied once per input in order; the spec's own
+// three-step run is tiny_sequence. Tap a step to change its input. Words are SAY_ and numbers K_
+// constants of specs/widgets/nn-rnn.t27.
+import { tinySequence, tinyStep } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, el, fmt, legend, plot, root, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const N = K('K_STEPS', 8)
+const keys = [...Array(N)].map((_, t) => `x${t}`)
+const presets = K('K_PRESET_INPUTS', [])
+const ctl = controls(keys.map((key) => ({ key, kind: 'hidden', min: -1, max: 1 })),
+ { names: W.SAY_PRESET_NAMES, values: Object.fromEntries(keys.map((k, t) => [k, (W.SAY_PRESET_NAMES ?? []).map((_, p) => presets[p * N + t])])) }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const row = el('div', { class: 'nn-steps', role: 'group', 'aria-label': say('SAY_TAP') })
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('p', { class: 'dk-label' }, say('SAY_TAP')), row, svg, legend([[say('SAY_X'), COLORS[2]], [say('SAY_H'), COLORS[0]]]), stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const xs = keys.map((k) => s[k])
+ const hs = []
+ let h = 0
+ for (const x of xs) hs.push((h = tinyStep(x, h)))
+ row.replaceChildren(...xs.map((x, t) => {
+ const b = el('button', { class: 't27-btn', type: 'button', 'aria-label': `${say('SAY_STEP')} ${t}` }, x > 0 ? '+1' : x < 0 ? '−1' : '0')
+ b.addEventListener('click', () => ctl.set(keys[t], x >= 1 ? -1 : x + 1))
+ return b
+ }))
+ svg.replaceChildren()
+ const p = plot(svg, { w: 640, h: 320, x0: -0.5, x1: N - 0.5, y0: -1.1, y1: 1.1, xticks: [...Array(N)].map((_, t) => t), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_STEP') })
+ xs.forEach((x, t) => svg.append(svgEl('rect', { x: p.x(t) - 14, y: Math.min(p.y(0), p.y(x)), width: 28, height: Math.max(2, Math.abs(p.y(x) - p.y(0))), rx: 5, fill: COLORS[2], opacity: 0.55 })))
+ p.line(hs.map((v, t) => [t, v]), COLORS[0], 3)
+ hs.forEach((v, t) => svg.append(svgEl('circle', { cx: p.x(t), cy: p.y(v), r: 6, fill: COLORS[0], stroke: '#020403', 'stroke-width': 2 })))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_H_LAST'), value: fmt(hs[N - 1], 4), tone: 0 },
+ { label: say('SAY_H_ALL'), value: hs.map((v) => fmt(v, 2)).join(' '), tone: 1 },
+ { label: say('SAY_SPEC_RUN'), value: [0, 1, 2].map((t) => fmt(tinySequence(t), 4)).join(' '), tone: 2 },
+ { label: say('SAY_ECHO'), value: [tinyStep(1, 0), tinyStep(0, tinyStep(1, 0))].map((v) => fmt(v, 4)).join(' '), tone: 5 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-sgd/card.png b/apps/website/public/widgets/nn-sgd/card.png
new file mode 100644
index 0000000000..2f28ed03bb
Binary files /dev/null and b/apps/website/public/widgets/nn-sgd/card.png differ
diff --git a/apps/website/public/widgets/nn-sgd/index.html b/apps/website/public/widgets/nn-sgd/index.html
new file mode 100644
index 0000000000..8c4a9c2e19
--- /dev/null
+++ b/apps/website/public/widgets/nn-sgd/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Gradient descent: how a network walks downhill · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Gradient descent: how a network walks downhill
+
Set the learning rate and momentum and watch a weight walk from 0 to the bottom of (x - 3)^2. Plain steps creep, momentum overshoots and rings, and a rate of 1 bounces forever.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-sgd/logic.js b/apps/website/public/widgets/nn-sgd/logic.js
new file mode 100644
index 0000000000..be989dea83
--- /dev/null
+++ b/apps/website/public/widgets/nn-sgd/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from sgd.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: d344b6438e73f6e275f3837c14267919d3629fa0e12898a96451e2782885884f
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-sgd/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABHwVgAAF9YAV/f39/fwBgAX8BfWACf38AYAR9fX9/AX0DCAcAAAECAgMEBAUBcAEBAQUDAQARBgkBfwFBgIDAAAsHXAYGbWVtb3J5AgAPdDI3X3ZhbGlkX2lucHV0AAAOdDI3X2RlY2F5X3N0ZXAAAQx0Mjdfb25lX3N0ZXAAAw90MjdfaW5pdF96ZXJvZXMABAt0MjdfZGVzY2VuZAAGCuAGBwcAQwrXIzwLjwECAX8BfSOAgICAAEEwayIAJICAgIAAIABBATYCFCAAQQA2AgwgAEGAgICABDYCCCAAQoCAgPgDNwIgIABCzZmz7gM3AhggAEEAOgAoIAAgAEEMajYCECAAQQA6ACwgAEEIakEBQYCAwIAAIABBEGogAEEYahCCgICAACAAKgIIIQEgAEEwaiSAgICAACABC70BAQZ9QwAAgD8gBCoCDJMhBSAEKgIIIQYgBCoCBCEHIAQqAgAhCCADKAIAIQMgBC0AEEEBcSEEA0ACQAJAIAFFDQAgAioCACAGIAAqAgCUkiEJIAdDAAAAAF5FDQEgAyAFIAmUIAcgAyoCAJSSIgo4AgACQCAEDQAgCiEJDAILIAkgByAKlJIhCQwBCw8LIAAgACoCACAIIAmUkzgCACADQQRqIQMgAEEEaiEAIAJBBGohAiABQX9qIQEMAAsLmwECAX8BfSOAgICAAEEwayIBJICAgIAAIAFBAjYCFCABQgA3AgggAUKAgID8g4CAgMAANwIAIAFC76SM1AM3AiAgAUKKro/h48yZsz83AhggAUEAOgAoIAEgAUEIajYCECABQQA6ACwgAUECQYSAwIAAIAFBEGogAUEYahCCgICAACABIABBAnRqKgIAIQIgAUEwaiSAgICAACACC20CAX8BfSOAgICAAEEgayIBJICAgIAAIAFBAzYCHCABQYCAgIcENgIUIAFCgICAh4SAgPDAADcCDCABIAFBDGo2AhggAUEYakEDEIWAgIAAIAFBDGogAEECdGoqAgAhAiABQSBqJICAgIAAIAILKwAgACgCACEAAkADQCABRQ0BIABBADYCACAAQQRqIQAgAUF/aiEBDAALCwvOAQIBfwF9I4CAgIAAQTBrIgQkgICAgAAgBEEBNgIQIARBgICAhQQ2AgggBEEANgIEIAQgBEEIajYCDCAEQQxqQQEQhYCAgABDAAAAACEFIAJBAXEhAgJAA0AgA0UNASAEIAVDAABAwJIiBSAFkjgCFCAEQgA3AiAgBCABOAIcIAQgADgCGCAEIAI6ACggBCACOgAsIARBBGpBASAEQRRqIARBDGogBEEYahCCgICAACADQX9qIQMgBCoCBCEFDAALCyAEQTBqJICAgIAAIAULCxUBAEGAgMAACwwAAAAAzczMPc3MTD4=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function descend(lr, momentum, nesterov, steps) {
+ return W.t27_descend(lr, momentum, nesterov ? 1 : 0, int(steps, 0, 4294967295, "descend.steps"));
+}
+function initZeroes(index) {
+ return W.t27_init_zeroes(int(index, 0, 4294967295, "init_zeroes.index"));
+}
+function oneStep(index) {
+ return W.t27_one_step(int(index, 0, 4294967295, "one_step.index"));
+}
+function decayStep() {
+ return W.t27_decay_step();
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+const __NOT_LOWERED__ = Object.freeze(["init", "update", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ decayStep,
+ descend,
+ initZeroes,
+ oneStep,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-sgd/logic.t27 b/apps/website/public/widgets/nn-sgd/logic.t27
new file mode 100644
index 0000000000..af7c998fa1
--- /dev/null
+++ b/apps/website/public/widgets/nn-sgd/logic.t27
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Nesterov accelerated gradient (NAG) | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: init called `new f32[n]`; update read undeclared locals; dampening and
+// nesterov were ignored; and weight decay was ADDED to the parameter
+// (`- v + wd * p`), which grows the weights it should shrink. update is now
+// PyTorch's SGD: g = grad + wd p; v = mu v + (1 - dampening) g;
+// step = g + mu v (Nesterov) or v; p = p - lr step.
+
+module Sgd;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_LR : f32 = 0.01;
+ const DEFAULT_MOMENTUM : f32 = 0.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SGDConfig = struct {
+ learning_rate : f32,
+ momentum : f32,
+ weight_decay : f32,
+ dampening : f32,
+ nesterov : bool,
+ };
+
+ pub const SGDState = struct {
+ velocity : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // init: the first param_count velocities start at zero (the caller owns the buffer)
+ fn init(state: SGDState, param_count: u32) -> void {
+ var i : usize = 0;
+ while (i < param_count) {
+ state.velocity[i] = 0.0;
+ i = i + 1;
+ }
+ }
+
+ fn update(params: []f32, grads: []const f32, state: SGDState, config: SGDConfig) -> void {
+ var i : usize = 0;
+ while (i < params.len) {
+ const g = grads[i] + config.weight_decay * params[i];
+ var step = g;
+ if (config.momentum > 0.0) {
+ state.velocity[i] = config.momentum * state.velocity[i] + (1.0 - config.dampening) * g;
+ if (config.nesterov) {
+ step = g + config.momentum * state.velocity[i];
+ } else {
+ step = state.velocity[i];
+ }
+ }
+ params[i] = params[i] - config.learning_rate * step;
+ i = i + 1;
+ }
+ }
+
+ fn cfg(lr: f32, momentum: f32, wd: f32, nesterov: bool) -> SGDConfig {
+ return SGDConfig{ .learning_rate = lr, .momentum = momentum, .weight_decay = wd, .dampening = 0.0, .nesterov = nesterov };
+ }
+
+ // Minimise f(x) = (x - 3)^2 (gradient 2 (x - 3)) from x = 0 for `steps` steps
+ fn descend(lr: f32, momentum: f32, nesterov: bool, steps: u32) -> f32 {
+ var params : [1]f32 = [0.0];
+ var velocity : [1]f32 = [5.0];
+ const state = SGDState{ .velocity = &velocity };
+ init(state, 1);
+ var n : u32 = 0;
+ while (n < steps) {
+ const grads : [1]f32 = [2.0 * (params[0] - 3.0)];
+ update(¶ms, &grads, state, cfg(lr, momentum, 0.0, nesterov));
+ n = n + 1;
+ }
+ return params[0];
+ }
+
+ fn init_zeroes(index: usize) -> f32 {
+ var velocity : [3]f32 = [7.0, 7.0, 7.0];
+ init(SGDState{ .velocity = &velocity }, 3);
+ return velocity[index];
+ }
+
+ // One step on params (1, 2), grads (0.1, 0.2), momentum 0.9, weight decay 0.001
+ fn one_step(index: usize) -> f32 {
+ var params : [2]f32 = [1.0, 2.0];
+ const grads : [2]f32 = [0.1, 0.2];
+ var velocity : [2]f32 = [0.0, 0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.01, 0.9, 0.001, false));
+ return params[index];
+ }
+
+ // A step with zero gradient and weight decay shrinks the weight toward 0
+ fn decay_step() -> f32 {
+ var params : [1]f32 = [2.0];
+ const grads : [1]f32 = [0.0];
+ var velocity : [1]f32 = [0.0];
+ update(¶ms, &grads, SGDState{ .velocity = &velocity }, cfg(0.1, 0.0, 0.5, false));
+ return params[0];
+ }
+
+ fn valid_input() -> f32 {
+ return DEFAULT_LR;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test "init_basic_case"
+ given first = init_zeroes(0)
+ when last = init_zeroes(2)
+ then first == 0.0 and last == 0.0
+
+ // v = g + wd p (momentum starts at 0): p0 = 1 - 0.01 (0.1 + 0.001) = 0.99899
+ test "update_basic_case"
+ given p0 = one_step(0)
+ when p1 = one_step(1)
+ then p0 != 1.0 and p1 != 2.0 and @abs(p0 - 0.99899) < 0.000001 and @abs(p1 - (2.0 - 0.01 * 0.202)) < 0.000001
+
+ // Weight decay pulls the weight toward 0: 2 - 0.1 * 0.5 * 2 = 1.9
+ test "update_weight_decay_shrinks"
+ given result = decay_step()
+ then @abs(result - 1.9) < 0.000001
+
+ // Plain, momentum and Nesterov SGD all reach the minimum at 3
+ test "update_descends_a_quadratic"
+ given plain = descend(0.1, 0.0, false, 100)
+ when heavy = descend(0.05, 0.9, false, 300)
+ then @abs(plain - 3.0) < 0.0001 and @abs(heavy - 3.0) < 0.0001 and @abs(descend(0.05, 0.9, true, 300) - 3.0) < 0.0001
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // learning_rate > 0
+ invariant sgd_constraint_0
+ assert valid_input() > 0.0
+
+ // 0 <= momentum < 1
+ invariant sgd_constraint_1
+ assert DEFAULT_MOMENTUM >= 0.0 and DEFAULT_MOMENTUM < 1.0
+
+ // weight_decay >= 0: a non-negative decay never grows a weight with zero gradient
+ invariant sgd_constraint_2
+ assert decay_step() < 2.0
+
+ // params.size() == grads.size(): every parameter moves
+ invariant sgd_constraint_3
+ assert one_step(0) != 1.0 and one_step(1) != 2.0
+
+ // If momentum > 0: state.velocity.size() == params.size() -- init zeroes all of it
+ invariant sgd_constraint_4
+ assert init_zeroes(1) == 0.0
diff --git a/apps/website/public/widgets/nn-sgd/nn-sgd.t27 b/apps/website/public/widgets/nn-sgd/nn-sgd.t27
new file mode 100644
index 0000000000..6efd6e901e
--- /dev/null
+++ b/apps/website/public/widgets/nn-sgd/nn-sgd.t27
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-sgd.t27 -- gradient descent with momentum and Nesterov
+; Source of truth for public/widgets/nn-sgd/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-sgd/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/sgd.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 16 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets lr from 0.01 to 1, momentum from 0 to 0.95, Nesterov and the number
+; of steps. Every point is descend(lr, momentum, nesterov, n), the spec minimising (x - 3)^2 from 0;
+; the dashed path is descend with momentum 0.
+
+; WHAT IT DOES NOT CLAIM: one weight on one bowl; a network has millions and its bowl is not a
+; parabola.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_sgd;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-sgd";
+pub const TITLE : str = "Gradient descent: how a network walks downhill";
+pub const DESCRIPTION : str = "Set the learning rate and momentum and watch a weight walk from 0 to the bottom of (x - 3)^2. Plain steps creep, momentum overshoots and rings, and a rate of 1 bounces forever.";
+pub const IMAGE_ALT : str = "SGD card: momentum 0.9 at lr 0.1 for 50 steps. The green path of dots shoots up from 0 past the white dashed minimum at 3 to about 5.1, swings back below it and rings in toward 3; the dashed violet plain path creeps up smoothly. Tiles read x after 50 steps 3.09149, plain 2.99996 and step 1 at 0.6000.";
+pub const HOOK : str = "lr 0.1 reaches 2.99996 in 50 steps. Add momentum 0.9 and it shoots to 5.13 first; at lr 1 it bounces between 0 and 6 for good.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/sgd.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 16 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS_HI : u8 = 100;
+pub const K_MINIMUM : u8 = 3;
+pub const K_LR_LO_C : u8 = 1;
+pub const K_LR_HI_C : u8 = 100;
+pub const K_MOM_HI_C : u8 = 95;
+pub const K_Y_LO : i8 = -1;
+pub const K_Y_HI : u8 = 7;
+pub const K_Y_TICKS : [5]i8 = [0, 2, 3, 4, 6];
+pub const K_STEP_TICKS : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_LR_C : [4]u8 = [10, 10, 10, 100];
+pub const K_PRESET_MOM_C : [4]u8 = [90, 0, 90, 0];
+pub const K_PRESET_NEST : [4]u8 = [0, 0, 1, 0];
+pub const K_PRESET_STEPS : [4]u8 = [50, 50, 50, 20];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Momentum 0.9", "Plain, lr 0.1", "Nesterov 0.9", "lr 1: it bounces"];
+pub const SAY_ASK : str = "How does a network walk downhill, and why does it overshoot?";
+pub const SAY_LR : str = "learning rate";
+pub const SAY_MOMENTUM : str = "momentum";
+pub const SAY_NESTEROV : str = "Nesterov";
+pub const SAY_STEPS : str = "steps";
+pub const SAY_PLOT_LABEL : str = "The weight after every step, from 0 toward the minimum at 3";
+pub const SAY_PATH : str = "this run";
+pub const SAY_PLAIN : str = "no momentum";
+pub const SAY_MIN : str = "the minimum";
+pub const SAY_X_NOW : str = "x after the steps";
+pub const SAY_X_PLAIN : str = "without momentum";
+pub const SAY_X_ONE : str = "after step 1";
+pub const SAY_NOTE : str = "Each step moves x by -lr times the gradient 2 (x - 3). Momentum adds 0.9 of the last move, which speeds up a long slope and overshoots a sharp bottom; Nesterov takes the gradient one step ahead.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/sgd.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_minimum_is_inside_the_axis {
+ assert K_MINIMUM == 3;
+ assert K_Y_LO < 0;
+ assert K_Y_HI > K_MINIMUM * 2;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_LR_C[3] == K_LR_HI_C;
+ assert K_PRESET_MOM_C[0] <= K_MOM_HI_C;
+ assert K_PRESET_STEPS[0] <= K_STEPS_HI;
+}
diff --git a/apps/website/public/widgets/nn-sgd/tool.js b/apps/website/public/widgets/nn-sgd/tool.js
new file mode 100644
index 0000000000..64989442c5
--- /dev/null
+++ b/apps/website/public/widgets/nn-sgd/tool.js
@@ -0,0 +1,42 @@
+// nn-sgd/tool.js -- lesson 16 of neural-networks: how a network walks downhill. Gradient descent
+// moves a weight against its gradient by lr every step; momentum keeps some of the last move, and
+// Nesterov looks one step ahead. Every point is descend(lr, momentum, nesterov, n) of gHashTag/t27
+// specs/ml/optimizer/sgd.t27, compiled to wasm (./logic.js): the spec minimises (x - 3)^2 from x = 0.
+// Words are SAY_ and numbers K_ constants of specs/widgets/nn-sgd.t27.
+import { descend } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const NMAX = K('K_STEPS_HI', 100)
+const MIN = K('K_MINIMUM', 3)
+const ctl = controls([
+ { key: 'lr', label: say('SAY_LR'), min: centi(K('K_LR_LO_C', 1)), max: centi(K('K_LR_HI_C', 100)), step: 0.01, fmt: (v) => fmt(v, 2) },
+ { key: 'mom', label: say('SAY_MOMENTUM'), min: 0, max: centi(K('K_MOM_HI_C', 95)), step: 0.05, fmt: (v) => fmt(v, 2) },
+ { key: 'nest', label: say('SAY_NESTEROV'), kind: 'toggle' },
+ { key: 'n', label: say('SAY_STEPS'), min: 1, max: NMAX },
+], { names: W.SAY_PRESET_NAMES, values: { lr: K('K_PRESET_LR_C', []).map(centi), mom: K('K_PRESET_MOM_C', []).map(centi), nest: K('K_PRESET_NEST', []), n: K('K_PRESET_STEPS', []) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_PATH'), COLORS[0]], [say('SAY_PLAIN'), COLORS[5], true], [say('SAY_MIN'), '#ffffff', true]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 340, x0: 0, x1: NMAX, y0: K('K_Y_LO', -1), y1: K('K_Y_HI', 7), xticks: K('K_STEP_TICKS', []), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_STEPS') }
+ const p = plot(svg, o)
+ svg.append(svgEl('line', { x1: p.left, x2: 640 - p.right, y1: p.clampY(MIN), y2: p.clampY(MIN), stroke: '#fff', 'stroke-dasharray': '6 6', opacity: 0.7 }))
+ const path = (lr, mom, nest, n) => [...Array(n + 1)].map((_, k) => [k, descend(lr, mom, nest, k)])
+ p.line(path(s.lr, 0, false, NMAX), COLORS[5], 1.5, { 'stroke-dasharray': '4 5' })
+ const run = path(s.lr, s.mom, !!s.nest, s.n)
+ p.line(run, COLORS[0], 3)
+ for (const [k, x] of run) svg.append(svgEl('circle', { cx: p.x(k), cy: p.clampY(x), r: s.n <= 30 ? 4 : 2, fill: COLORS[0] }))
+ cursor(svg, p, s.n, [[descend(s.lr, s.mom, !!s.nest, s.n), COLORS[1]]])
+ dragX(svg, p, o, (n) => ctl.set('n', Math.round(n)))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_X_NOW'), value: fmt(descend(s.lr, s.mom, !!s.nest, s.n), 5), tone: 0 },
+ { label: say('SAY_X_PLAIN'), value: fmt(descend(s.lr, 0, false, s.n), 5), tone: 5 },
+ { label: say('SAY_X_ONE'), value: fmt(descend(s.lr, s.mom, !!s.nest, 1), 4), tone: 1 },
+ { label: say('SAY_MIN'), value: fmt(MIN, 0), tone: 2 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-sigmoid/card.png b/apps/website/public/widgets/nn-sigmoid/card.png
new file mode 100644
index 0000000000..d9ff4df0be
Binary files /dev/null and b/apps/website/public/widgets/nn-sigmoid/card.png differ
diff --git a/apps/website/public/widgets/nn-sigmoid/index.html b/apps/website/public/widgets/nn-sigmoid/index.html
new file mode 100644
index 0000000000..695b90c6f9
--- /dev/null
+++ b/apps/website/public/widgets/nn-sigmoid/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Sigmoid: turn any number into a probability · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Sigmoid: turn any number into a probability
+
Drag along the S-curve: every input lands between 0 and 1, and the slope peaks at 0.25 in the middle and dies away at both ends.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-sigmoid/logic.js b/apps/website/public/widgets/nn-sigmoid/logic.js
new file mode 100644
index 0000000000..a5ccfc4881
--- /dev/null
+++ b/apps/website/public/widgets/nn-sigmoid/logic.js
@@ -0,0 +1,27 @@
+// Generated by scripts/t27-logic.mjs from sigmoid_activation.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: b8873ed230ede07279ef5a3238d93e9cfe7e5c34904a7e769deee7131e06e71f
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-sigmoid/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABEANgAAF/YAF9AX1gAn1/AX0DBgUAAQEBAgQFAXABAQEFAwEAEQYJAX8BQYCAwAALBzsEBm1lbW9yeQIAD3QyN19jb25maWdfc2l6ZQAADnQyN19kZXJpdmF0aXZlAAELdDI3X2ZvcndhcmQAAgrTBQUEAEEACw0AIABDAACAPyAAk5QLQAEBfQJAAkAgAEMAAAAAYEUNAEMAAIA/IQEgAIwQg4CAgAAhAAwBCyAAEIOAgIAAIgAhAQsgASAAQwAAgD+SlQvUAgIDfwN9AkACQCAAIABcDQAgALwiAUEfdiECAkACQAJAAkACQAJAAkAgAUH/////B3EiA0HP2LqVBEsNACADQZjkxfUDTQ0BIANBkquU/ANNDQIMBAsgA0GAgID8B0sNBgJAIAFBAEgNACADQZjkxZUESQ0AIABDAAAAf5QPCyABQX9KDQNDAAAAACEEIANBtOO/lgRLDQcMAwsgA0GAgIDIA00NAUEAIQNDAAAAACEFIAAhBAwECyACQQFzIAJrIQMMAgsgAEMAAIA/kg8LIABDO6q4P5QgAkECdCoCgIDAgACS/AAhAwsgACADsiIEQwByMb+UkiIAIARDjr6/NZQiBZMhBAsgACAEIAQgBCAElCIGIAZDFVI1u5RDj6oqPpKUkyIGlEMAAABAIAaTlSAFk5JDAACAP5IhBCADRQ0BIAQgAxCEgICAAA8LIAAhBAsgBAumAgEEfwJAAkAgACAAXA0AIAC8IgJB/////wdxQYCAgPwHTw0AIAJBgICAgHhxIQNBCCACQQF0IgRnayAEQRh2IARBgICACEkbIQQCQCABQQBIDQACQCABQf4BIARrTA0AQwAAgH8gAJgPCwJAIAQgAWoiBUEASg0AIAIgAXQgA3K+DwsCQCAEQQBKDQAgAkEBIARrdEH///8DcSAFQRd0ciADcr4PCyABQRd0IAJqvg8LIAFBACAEa0oNAQJAIAFBaSAEa04NACADvg8LQYCAgARBACAEQQBKGyACQf///wNxckEAIAFrIgNBASABIARqayIBIAMgAUgbQX9qIgF2QQFqQQF2IAJoIAFBH3FGQX9zcb4gAJghAAsgAA8LIAFBF3QgAmq+CwsRAQBBgIDAAAsIAAAAPwAAAL8=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function forward(x) {
+ return W.t27_forward(x);
+}
+function derivative(sigmoidX) {
+ return W.t27_derivative(sigmoidX);
+}
+function configSize() {
+ return W.t27_config_size() >>> 0;
+}
+const __NOT_LOWERED__ = Object.freeze(["forward_batch"]);
+export {
+ __NOT_LOWERED__,
+ configSize,
+ derivative,
+ forward
+};
diff --git a/apps/website/public/widgets/nn-sigmoid/logic.t27 b/apps/website/public/widgets/nn-sigmoid/logic.t27
new file mode 100644
index 0000000000..4e6d84ddb3
--- /dev/null
+++ b/apps/website/public/widgets/nn-sigmoid/logic.t27
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Derivative (compute from output) | φ² + 1/φ² = 3 | TRINITY
+
+module Sigmoid;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SigmoidConfig = struct {
+ none : void,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // forward(x: f32) → f32
+ fn forward(x: f32) -> f32 {
+ // Numerically stable split to avoid overflow:
+ // For x >= 0: 1 / (1 + exp(-x))
+ // For x < 0: exp(x) / (1 + exp(x))
+ // Both branches are mathematically equivalent but the second avoids exp(large positive)
+ if (x >= 0.0) {
+ return 1.0 / (1.0 + @exp(-x));
+ } else {
+ return @exp(x) / (1.0 + @exp(x));
+ }
+ }
+
+ // forward_batch(input: []f32) → []f32
+ fn forward_batch(input: []f32) -> []f32 {
+ // In-place map over input: alias input and return it
+ // This is safe because we're only reading from input and writing to the same locations
+ let i = 0;
+ while (i < input.len) {
+ input[i] = forward(input[i]);
+ i += 1;
+ }
+ return input;
+ }
+
+ // derivative(sigmoid_x: f32) → f32
+ fn derivative(sigmoid_x: f32) -> f32 {
+ // Derivative computed from output: σ'(x) = σ(x) * (1 - σ(x))
+ return sigmoid_x * (1.0 - sigmoid_x);
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // #8980: through a function, so the check runs and is counted (it folded at compile
+ // time before, and the report counted it as a vacuous pass).
+ fn config_size() -> usize {
+ return @sizeOf(SigmoidConfig);
+ }
+
+ test sigmoid_has_no_tunable_parameter
+ given size = config_size()
+ then size == 0
+
+ // sigmoid_config_is_a_placeholder_not_an_empty_struct read std.meta.fields, which t27
+ // cannot lower; the parser discarded it. The test above already pins @sizeOf == 0.
+
+ test forward_at_zero_is_one_half
+ given y = forward(0.0)
+ then @abs(y - 0.5) < 1e-6
+
+ test forward_confined_to_unit_interval
+ given pos_large = forward(10.0)
+ given neg_large = forward(-10.0)
+ then (pos_large > 0.0) and (pos_large < 1.0) and (neg_large > 0.0) and (neg_large < 1.0)
+
+ test derivative_at_zero_is_maximum
+ given dy = derivative(forward(0.0))
+ then @abs(dy - 0.25) < 1e-6
+
+ test forward_batch_preserves_length
+ given input = []f32{0.0, 1.0, -1.0, 2.0, -2.0}
+ given result = forward_batch(input)
+ then result.len == input.len
+
+ // sigmoid(-x) = 1 - sigmoid(x); sigmoid(2) = 0.8807971
+ test forward_is_symmetric_about_one_half
+ given up = forward(2.0)
+ when down = forward(-2.0)
+ then @abs(up + down - 1.0) < 0.000001 and @abs(up - 0.8807971) < 0.000001
+
+ // The negative branch never computes exp(+x): no overflow at -100
+ test forward_far_left_is_zero_not_nan
+ given y = forward(-100.0)
+ then y >= 0.0 and y < 0.000001
diff --git a/apps/website/public/widgets/nn-sigmoid/nn-sigmoid.t27 b/apps/website/public/widgets/nn-sigmoid/nn-sigmoid.t27
new file mode 100644
index 0000000000..f88b67dbf7
--- /dev/null
+++ b/apps/website/public/widgets/nn-sigmoid/nn-sigmoid.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-sigmoid.t27 -- the sigmoid: any number into a probability
+; Source of truth for public/widgets/nn-sigmoid/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-sigmoid/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/sigmoid_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 2 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -8 to 8. The curve is forward and the slope derivative of
+; the spec, which takes the OUTPUT of the sigmoid, as backpropagation does; the tiles read the slope
+; at x, at 0 and at the far right.
+
+; WHAT IT DOES NOT CLAIM: the slope bound of 0.25 is read off the compiled curve at x = 0, not
+; proved here.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_sigmoid;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-sigmoid";
+pub const TITLE : str = "Sigmoid: turn any number into a probability";
+pub const DESCRIPTION : str = "Drag along the S-curve: every input lands between 0 and 1, and the slope peaks at 0.25 in the middle and dies away at both ends.";
+pub const IMAGE_ALT : str = "Sigmoid card at x = 2: the green S-curve from 0 to 1 over -8..8, the dashed gold slope a bump peaking at 0.25 at x = 0. Tiles read sigmoid 0.8808, slope 0.1050, slope at 0 = 0.2500 and slope at 8 = 0.000335; two bars repeat the output and the slope.";
+pub const HOOK : str = "1 / (1 + e^-x) squeezes the whole line into 0..1. Its slope is never above 0.25, which is why deep sigmoid nets learned slowly.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/sigmoid_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 2 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -800;
+pub const K_X_HI_C : i16 = 800;
+pub const K_X_STEP_C : u8 = 10;
+pub const K_X_TICKS : [5]i8 = [-8, -4, 0, 4, 8];
+pub const K_Y_TICKS_C : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_X_C : [4]i16 = [200, 0, -400, 600];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["x = 2", "x = 0", "x = -4", "x = 6"];
+pub const SAY_ASK : str = "How does a neuron turn any number into a probability?";
+pub const SAY_X : str = "input x";
+pub const SAY_F : str = "sigmoid(x)";
+pub const SAY_DF : str = "slope";
+pub const SAY_PLOT_LABEL : str = "The sigmoid of x and its slope from -8 to 8, with a cursor you drag";
+pub const SAY_BARS_LABEL : str = "The output and the slope at x as bars from 0 to 1";
+pub const SAY_OUT : str = "probability";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_AT_ZERO : str = "slope at 0";
+pub const SAY_FAR : str = "slope at 8";
+pub const SAY_NOTE : str = "The slope is s (1 - s), computed from the output s itself. Far from 0 it is close to nothing, so a saturated sigmoid passes almost no gradient back.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/sigmoid_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_range_is_symmetric {
+ assert K_X_LO_C + K_X_HI_C == 0;
+ assert K_X_HI_C == 800;
+}
+
+test the_axis_spans_zero_to_one {
+ assert K_Y_TICKS_C[0] == 0;
+ assert K_Y_TICKS_C[4] == 100;
+ assert K_PRESET_X_C[1] == 0;
+}
diff --git a/apps/website/public/widgets/nn-sigmoid/tool.js b/apps/website/public/widgets/nn-sigmoid/tool.js
new file mode 100644
index 0000000000..999f4a3ff7
--- /dev/null
+++ b/apps/website/public/widgets/nn-sigmoid/tool.js
@@ -0,0 +1,36 @@
+// nn-sigmoid/tool.js -- lesson 2 of neural-networks: how a neuron turns any number into a
+// probability. The curve, its slope and every tile are forward and derivative of gHashTag/t27
+// specs/ml/activation/sigmoid_activation.t27, compiled to wasm (./logic.js); the slope is computed
+// from the output, as the spec does. Words are SAY_ and numbers K_ constants of specs/widgets/nn-sigmoid.t27.
+import { derivative, forward } from './logic.js'
+import { COLORS, K, W, ask, bars, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const X0 = centi(K('K_X_LO_C', -800)), X1 = centi(K('K_X_HI_C', 800))
+const ctl = controls([
+ { key: 'x', label: say('SAY_X'), min: X0, max: X1, step: centi(K('K_X_STEP_C', 10)), fmt: (v) => fmt(v, 1) },
+], { names: W.SAY_PRESET_NAMES, values: { x: K('K_PRESET_X_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const bar = canvas('nn-bars', say('SAY_BARS_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_F'), COLORS[0]], [say('SAY_DF'), COLORS[1], true]]), bar, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 320, x0: X0, x1: X1, y0: 0, y1: 1, xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS_C', []).map(centi), xlabel: say('SAY_X') }
+ const p = plot(svg, o)
+ p.line(sample((x) => forward(x), X0, X1), COLORS[0], 3)
+ p.line(sample((x) => derivative(forward(x)), X0, X1), COLORS[1], 2, { 'stroke-dasharray': '6 5' })
+ const y = forward(s.x)
+ cursor(svg, p, s.x, [[y, COLORS[0]], [derivative(y), COLORS[1]]])
+ dragX(svg, p, o, (x) => ctl.set('x', x))
+ bars(bar, { values: [y, derivative(y)], labels: [say('SAY_F'), say('SAY_DF')], colors: [COLORS[0], COLORS[1]], lo: 0, hi: 1, h: 200, ticks: K('K_Y_TICKS_C', []).map(centi), fmt: (v) => fmt(v, 4) })
+ stat.replaceChildren(tiles([
+ { label: say('SAY_OUT'), value: fmt(y, 4), tone: 0 },
+ { label: say('SAY_GRAD'), value: fmt(derivative(y), 4), tone: 1 },
+ { label: say('SAY_AT_ZERO'), value: fmt(derivative(forward(0)), 4), tone: 2 },
+ { label: say('SAY_FAR'), value: fmt(derivative(forward(X1)), 6), tone: 6 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-silu/card.png b/apps/website/public/widgets/nn-silu/card.png
new file mode 100644
index 0000000000..065dfe534e
Binary files /dev/null and b/apps/website/public/widgets/nn-silu/card.png differ
diff --git a/apps/website/public/widgets/nn-silu/index.html b/apps/website/public/widgets/nn-silu/index.html
new file mode 100644
index 0000000000..8b5b74410c
--- /dev/null
+++ b/apps/website/public/widgets/nn-silu/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Swish: one knob between a smooth curve and ReLU · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Swish: one knob between a smooth curve and ReLU
+
Drag along x times sigmoid(beta x). At beta 1 it is SiLU, with its dip of -0.28; turn beta up and the curve closes on ReLU, turn it down and it flattens toward x / 2.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-silu/logic.js b/apps/website/public/widgets/nn-silu/logic.js
new file mode 100644
index 0000000000..205aa9b157
--- /dev/null
+++ b/apps/website/public/widgets/nn-silu/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from silu_swish_activation.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 407115e99a8dd8f07ec28f899ef3336b0b9106ce30581cb0a3ed4217f5962364
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-silu/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABHgVgAAF9YAR9fX1/AX1gAn19AX1gAX0BfWACfX8BfQMIBwABAgIDAwQEBQFwAQEBBQMBABEGCQF/AUGAgMAACwdYBgZtZW1vcnkCAA90MjdfdmFsaWRfaW5wdXQAAAx0MjdfYmF0Y2hfYXQAAQ50MjdfZGVyaXZhdGl2ZQACC3QyN19mb3J3YXJkAAMLdDI3X3NpZ21vaWQABAr6BgcHAEMAAABAC40BAQN/I4CAgIAAQRBrIgQkgICAgAAgBCACOAIMIAQgATgCCCAEIAA4AgRBACEFAkADQCAFQQxGDQEgBEEEaiAFaiIGKgIAIQIgBiACQwAAgD8gAowQhYCAgABDAACAP5KVlDgCACAFQQRqIQUMAAsLIARBBGogA0ECdGoqAgAhAiAEQRBqJICAgIAAIAILLQBDAACAPyAAIAGUIgCMEIWAgIAAQwAAgD+SlSIBIAAgAZRDAACAPyABk5SSCx0AIABDAACAPyABIACMlBCFgICAAEMAAIA/kpWUCxcAQwAAgD8gAIwQhYCAgABDAACAP5KVC9QCAgN/A30CQAJAIAAgAFwNACAAvCIBQR92IQICQAJAAkACQAJAAkACQCABQf////8HcSIDQc/YupUESw0AIANBmOTF9QNNDQEgA0GSq5T8A00NAgwECyADQYCAgPwHSw0GAkAgAUEASA0AIANBmOTFlQRJDQAgAEMAAAB/lA8LIAFBf0oNA0MAAAAAIQQgA0G047+WBEsNBwwDCyADQYCAgMgDTQ0BQQAhA0MAAAAAIQUgACEEDAQLIAJBAXMgAmshAwwCCyAAQwAAgD+SDwsgAEM7qrg/lCACQQJ0KgKAgMCAAJL8ACEDCyAAIAOyIgRDAHIxv5SSIgAgBEOOvr81lCIFkyEECyAAIAQgBCAEIASUIgYgBkMVUjW7lEOPqio+kpSTIgaUQwAAAEAgBpOVIAWTkkMAAIA/kiEEIANFDQEgBCADEIaAgIAADwsgACEECyAEC6YCAQR/AkACQCAAIABcDQAgALwiAkH/////B3FBgICA/AdPDQAgAkGAgICAeHEhA0EIIAJBAXQiBGdrIARBGHYgBEGAgIAISRshBAJAIAFBAEgNAAJAIAFB/gEgBGtMDQBDAACAfyAAmA8LAkAgBCABaiIFQQBKDQAgAiABdCADcr4PCwJAIARBAEoNACACQQEgBGt0Qf///wNxIAVBF3RyIANyvg8LIAFBF3QgAmq+DwsgAUEAIARrSg0BAkAgAUFpIARrTg0AIAO+DwtBgICABEEAIARBAEobIAJB////A3FyQQAgAWsiA0EBIAEgBGprIgEgAyABSBtBf2oiAXZBAWpBAXYgAmggAUEfcUZBf3NxviAAmCEACyAADwsgAUEXdCACar4LCxEBAEGAgMAACwgAAAA/AAAAvw==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function sigmoid(z) {
+ return W.t27_sigmoid(z);
+}
+function forward(x, beta) {
+ return W.t27_forward(x, beta);
+}
+function derivative(x, beta) {
+ return W.t27_derivative(x, beta);
+}
+function batchAt(a, b, c, index) {
+ return W.t27_batch_at(a, b, c, int(index, 0, 4294967295, "batch_at.index"));
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+const __NOT_LOWERED__ = Object.freeze(["forward_batch"]);
+export {
+ __NOT_LOWERED__,
+ batchAt,
+ derivative,
+ forward,
+ sigmoid,
+ validInput
+};
diff --git a/apps/website/public/widgets/nn-silu/logic.t27 b/apps/website/public/widgets/nn-silu/logic.t27
new file mode 100644
index 0000000000..404271397f
--- /dev/null
+++ b/apps/website/public/widgets/nn-silu/logic.t27
@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// SiLU derivative | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward_batch returned its input unchanged, beta was never read, and the
+// invariants asserted `true` over an undeclared `valid_input`. SiLU is now
+// x * sigmoid(beta * x) (Swish; beta = 1 is SiLU) on scalars and in place on slices.
+
+module SiluSwish;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const DEFAULT_BETA : f32 = 1.0;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SiLUConfig = struct {
+ beta : f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // sigmoid(z) = 1 / (1 + exp(-z))
+ fn sigmoid(z: f32) -> f32 {
+ return 1.0 / (1.0 + exp(-z));
+ }
+
+ // Swish(x) = x * sigmoid(beta * x); beta = 1 is SiLU
+ fn forward(x: f32, beta: f32) -> f32 {
+ return x * sigmoid(beta * x);
+ }
+
+ // forward on each element IN PLACE; returns the same slice
+ fn forward_batch(input: []f32, beta: f32) -> []f32 {
+ var i : usize = 0;
+ while (i < input.len) {
+ input[i] = forward(input[i], beta);
+ i = i + 1;
+ }
+ return input;
+ }
+
+ // d/dx x sigmoid(beta x) = s + beta x s (1 - s), s = sigmoid(beta x)
+ fn derivative(x: f32, beta: f32) -> f32 {
+ const s = sigmoid(beta * x);
+ return s + beta * x * s * (1.0 - s);
+ }
+
+ fn batch_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ var input : [3]f32 = [a, b, c];
+ const output = forward_batch(&input, DEFAULT_BETA);
+ return output[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 2.0;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ test "forward_basic_case"
+ given input = 0.0, beta = 1.0
+ when result = forward(input, beta)
+ then result == 0.0 // SiLU(0) = 0 * sigmoid(0) = 0
+
+ test "forward_positive_case"
+ given input = 1.0, beta = 1.0
+ when result = forward(input, beta)
+ then result > 0.0 // SiLU(1) = 1 * sigmoid(1) > 0
+
+ test "forward_negative_case"
+ given input = -1.0, beta = 1.0
+ when result = forward(input, beta)
+ then result < 0.0 // SiLU(-1) = -1 * sigmoid(-1) < 0
+
+ test "forward_batch_basic_case"
+ given beta = 1.0
+ when result = batch_at(1.0, 2.0, 3.0, 2)
+ then @abs(result - forward(3.0, beta)) < 0.0000001 and @abs(batch_at(1.0, 2.0, 3.0, 0) - 0.7310586) < 0.000001
+
+ test "forward_batch_zero_case"
+ given beta = 1.0
+ when result = batch_at(0.0, 0.0, 0.0, 0)
+ then result == 0.0 and batch_at(0.0, 0.0, 0.0, 1) == 0.0 and batch_at(0.0, 0.0, 0.0, 2) == 0.0 // SiLU(0) = 0
+
+ test "derivative_basic_case"
+ given input = 0.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU'(0) = sigmoid(0) + 0 * sigmoid(0) * (1 - sigmoid(0)) = sigmoid(0) > 0
+
+ test "derivative_positive_case"
+ given input = 1.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU derivative should be positive for positive inputs
+
+ test "derivative_negative_case"
+ given input = -1.0, beta = 1.0
+ when result = derivative(input, beta)
+ then result > 0.0 // SiLU derivative should be positive for negative inputs too
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // beta > 0
+ invariant silu_swish_constraint_0
+ assert DEFAULT_BETA > 0.0
+
+ // name: silu_formula -- x * sigmoid(x), so SiLU(x) - SiLU(-x) = x
+ invariant silu_swish_constraint_1
+ assert @abs(forward(valid_input(), DEFAULT_BETA) - forward(-valid_input(), DEFAULT_BETA) - valid_input()) < 0.000001
+
+ // name: derivative -- sigmoid(0) = 0.5 at the origin
+ invariant silu_swish_constraint_2
+ assert derivative(0.0, DEFAULT_BETA) == 0.5
+
+ // "Elfwing et al. (2017) - Sigmoid-Weighted Linear Units for Neural Network Function Approximation":
+ // SiLU dips below zero left of the origin
+ invariant silu_swish_constraint_3
+ assert forward(-valid_input(), DEFAULT_BETA) < 0.0
+
+ // "Ramachandran et al. (2017) - Searching for Activation Functions (Swish)": a large beta
+ // approaches ReLU
+ invariant silu_swish_constraint_4
+ assert @abs(forward(valid_input(), 50.0) - valid_input()) < 0.000001
diff --git a/apps/website/public/widgets/nn-silu/nn-silu.t27 b/apps/website/public/widgets/nn-silu/nn-silu.t27
new file mode 100644
index 0000000000..087ee92a09
--- /dev/null
+++ b/apps/website/public/widgets/nn-silu/nn-silu.t27
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-silu.t27 -- Swish and SiLU: one knob between a smooth curve and ReLU
+; Source of truth for public/widgets/nn-silu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-silu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/silu_swish_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 5 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -5 to 5 and sets beta from 0.25 to 8. The green curve is
+; forward(x, beta), the dotted one forward(x, 1), SiLU, and the dashed one derivative(x, beta).
+
+; WHAT IT DOES NOT CLAIM: the claim that large beta approaches ReLU is what the curves show on this
+; window, not a limit taken here.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_silu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-silu";
+pub const TITLE : str = "Swish: one knob between a smooth curve and ReLU";
+pub const DESCRIPTION : str = "Drag along x times sigmoid(beta x). At beta 1 it is SiLU, with its dip of -0.28; turn beta up and the curve closes on ReLU, turn it down and it flattens toward x / 2.";
+pub const IMAGE_ALT : str = "Swish card at beta 1, x = -1.28: the green curve dips to -0.2785 and then rises along the diagonal, over a dotted violet SiLU reference that matches it; the dashed gold slope crosses zero at the cursor. Tiles read output -0.2785, slope -0.0003, SiLU -0.2785 and the spec batch (-2, 0, 2) as -0.238 0.000 1.762.";
+pub const HOOK : str = "SiLU(-1.28) = -0.2785, its lowest point. At beta 8 the dip shrinks to almost nothing and the curve is ReLU with rounded corners.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/silu_swish_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 5 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -500;
+pub const K_X_HI_C : i16 = 500;
+pub const K_X_STEP_C : u8 = 2;
+pub const K_Y_LO_C : i16 = -100;
+pub const K_Y_HI_C : i16 = 500;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [7]i8 = [-1, 0, 1, 2, 3, 4, 5];
+pub const K_BETA_LO_C : u8 = 25;
+pub const K_BETA_HI_C : u16 = 800;
+pub const K_BETA_STEP_C : u8 = 25;
+pub const K_FIXTURE_C : [3]i16 = [-200, 0, 200];
+pub const K_PRESET_X_C : [4]i16 = [-128, -128, 200, 200];
+pub const K_PRESET_BETA_C : [4]u16 = [100, 400, 50, 800];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["SiLU at its dip", "beta 4 at the same x", "beta 0.5 at x = 2", "beta 8 at x = 2"];
+pub const SAY_ASK : str = "Is there one knob between a smooth curve and ReLU?";
+pub const SAY_X : str = "input x";
+pub const SAY_BETA : str = "beta";
+pub const SAY_F : str = "x sigmoid(beta x)";
+pub const SAY_SILU : str = "SiLU, beta 1";
+pub const SAY_DF : str = "slope";
+pub const SAY_PLOT_LABEL : str = "Swish of x for the chosen beta, SiLU for reference, and the slope";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_SILU_AT : str = "SiLU at x";
+pub const SAY_BATCH : str = "spec batch -2 0 2";
+pub const SAY_NOTE : str = "Swish multiplies x by a sigmoid gate of itself. beta sets how sharp the gate is: the same unit can be gentle or close to a switch.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/silu_swish_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test beta_one_is_silu {
+ assert K_PRESET_BETA_C[0] == 100;
+ assert K_BETA_LO_C < K_PRESET_BETA_C[0];
+ assert K_PRESET_BETA_C[3] == K_BETA_HI_C;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_X_C[0] >= K_X_LO_C;
+ assert K_PRESET_X_C[3] <= K_X_HI_C;
+ assert K_BETA_HI_C % K_BETA_STEP_C == 0;
+}
diff --git a/apps/website/public/widgets/nn-silu/tool.js b/apps/website/public/widgets/nn-silu/tool.js
new file mode 100644
index 0000000000..a6a7944f19
--- /dev/null
+++ b/apps/website/public/widgets/nn-silu/tool.js
@@ -0,0 +1,37 @@
+// nn-silu/tool.js -- lesson 5 of neural-networks: one knob between a smooth curve and ReLU. Swish is
+// x times sigmoid(beta x): at beta 1 it is SiLU, and as beta grows it closes on ReLU. Both curves
+// and every tile are forward and derivative of gHashTag/t27 specs/ml/activation/
+// silu_swish_activation.t27, compiled to wasm (./logic.js). Words are SAY_ and numbers K_ constants
+// of specs/widgets/nn-silu.t27.
+import { batchAt, derivative, forward } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const X0 = centi(K('K_X_LO_C', -500)), X1 = centi(K('K_X_HI_C', 500))
+const ctl = controls([
+ { key: 'x', label: say('SAY_X'), min: X0, max: X1, step: centi(K('K_X_STEP_C', 2)), fmt: (v) => fmt(v, 2) },
+ { key: 'beta', label: say('SAY_BETA'), min: centi(K('K_BETA_LO_C', 25)), max: centi(K('K_BETA_HI_C', 800)), step: centi(K('K_BETA_STEP_C', 25)), fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { x: K('K_PRESET_X_C', []).map(centi), beta: K('K_PRESET_BETA_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_F'), COLORS[0]], [say('SAY_SILU'), COLORS[5], true], [say('SAY_DF'), COLORS[1], true]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 340, x0: X0, x1: X1, y0: centi(K('K_Y_LO_C', -100)), y1: centi(K('K_Y_HI_C', 500)), xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_X') }
+ const p = plot(svg, o)
+ p.line(sample((x) => forward(x, 1), X0, X1), COLORS[5], 1.5, { 'stroke-dasharray': '4 5' })
+ p.line(sample((x) => derivative(x, s.beta), X0, X1), COLORS[1], 2, { 'stroke-dasharray': '6 5' })
+ p.line(sample((x) => forward(x, s.beta), X0, X1), COLORS[0], 3.5)
+ cursor(svg, p, s.x, [[forward(s.x, s.beta), COLORS[0]], [derivative(s.x, s.beta), COLORS[1]]])
+ dragX(svg, p, o, (x) => ctl.set('x', x))
+ const fx = K('K_FIXTURE_C', []).map(centi)
+ stat.replaceChildren(tiles([
+ { label: say('SAY_OUT'), value: fmt(forward(s.x, s.beta), 4), tone: 0 },
+ { label: say('SAY_GRAD'), value: fmt(derivative(s.x, s.beta), 4), tone: 1 },
+ { label: say('SAY_SILU_AT'), value: fmt(forward(s.x, 1), 4), tone: 5 },
+ { label: say('SAY_BATCH'), value: [0, 1, 2].map((i) => fmt(batchAt(fx[0], fx[1], fx[2], i), 3)).join(' '), tone: 2 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-softmax/card.png b/apps/website/public/widgets/nn-softmax/card.png
new file mode 100644
index 0000000000..3a1e8f7151
Binary files /dev/null and b/apps/website/public/widgets/nn-softmax/card.png differ
diff --git a/apps/website/public/widgets/nn-softmax/index.html b/apps/website/public/widgets/nn-softmax/index.html
new file mode 100644
index 0000000000..3a67c1c163
--- /dev/null
+++ b/apps/website/public/widgets/nn-softmax/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+Softmax: how a network turns scores into a choice · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
Softmax: how a network turns scores into a choice
+
Move three scores and the temperature: softmax turns them into probabilities that sum to one. Cool it and the top score takes almost everything; heat it and the choice evens out.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-softmax/logic.js b/apps/website/public/widgets/nn-softmax/logic.js
new file mode 100644
index 0000000000..969a633d49
--- /dev/null
+++ b/apps/website/public/widgets/nn-softmax/logic.js
@@ -0,0 +1,40 @@
+// Generated by scripts/t27-logic.mjs from softmax.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 7466b7f756ea132c84d11d53f7f71ca4a66aca69ca98feb2a58ff204157baa78
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-softmax/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABMwhgAAF9YAR9fX1/AX1gA39/fwBgA319fQF9YAV9fX19fwF9YAF/AX1gAX0BfWACfX8BfQMLCgABAgIDBAUEBgcEBQFwAQEBBQMBABEGCQF/AUGAgMAACwdcBgZtZW1vcnkCAA90MjdfdmFsaWRfaW5wdXQAAAt0MjdfZ3JhZF9hdAABDHQyN193ZWlnaHRlZAAED3QyN19iYWNrd2FyZF9hdAAGDnQyN19zb2Z0bWF4X2F0AAcK7wsKBwBDAABAQAvOAQEBfyOAgICAAEHQAGsiBCSAgICAACAEQQhqQQhqQQA2AgAgBCACOAIgIAQgATgCHCAEIAA4AhggBEIANwMIIARCgICA/AM3AiQgBEEBOgAsIARBAToAQCAEQRhqIARBCGogBEEkahCCgICAACAEQTBqQQhqQQA2AgAgBEIANwMwIARCgICA/AM3AkAgBEEBOgBIIARBAToATCAEQQhqIARBMGogBEHAAGoQg4CAgAAgBEEwaiADQQJ0aioCACECIARB0ABqJICAgIAAIAILswEBBH0gACoCACEDIAIqAgAhBEEAIQIDQAJAIAJBDEcNAEMAAAAAIQVBACECA0ACQCACQQxHDQBBACECAkADQCACQQxGDQEgASACaiIAIAAqAgAgBZU4AgAgAkEEaiECDAALCw8LIAEgAmogACACaioCACADkyAElRCIgICAACIGOAIAIAJBBGohAiAFIAaSIQUMAAsLIAAgAmoqAgAiBSADIAUgA14bIQMgAkEEaiECDAALC4QBAQJ9IAIqAgAhA0MAAAAAIQRBACECA0ACQCACQQxHDQBBACECAkADQCACQQxGDQEgASACaiAAIAJqKgIAIAJBgIDAgABqKgIAIASTlCADlTgCACACQQRqIQIMAAsLDwsgBCACQYCAwIAAaioCACAAIAJqKgIAlJIhBCACQQRqIQIMAAsLTwAgACABIAJDAACAP0EAEIWAgIAAQ83MzD2UIAAgASACQwAAgD9BARCFgICAAEPNzEw+lJIgACABIAJDAACAP0ECEIWAgIAAQ5qZmT6UkguJAQEBfyOAgICAAEEwayIFJICAgIAAIAVBGGpBADYCACAFIAI4AgwgBSABOAIIIAUgADgCBCAFQgA3AxAgBUEANgIkIAUgAzgCICAFQQE6ACggBUEBOgAsIAVBBGogBUEQaiAFQSBqEIKAgIAAIAVBEGogBEECdGoqAgAhAiAFQTBqJICAgIAAIAILbgIBfwF9I4CAgIAAQSBrIgEkgICAgAAgAUEIakEANgIAIAFCADcDACABQoCAgPwDNwIQIAFBAToAGCABQQE6ABxBjIDAgAAgASABQRBqEIOAgIAAIAEgAEECdGoqAgAhAiABQSBqJICAgIAAIAILEgAgACABIAIgAyAEEIWAgIAAC9QCAgN/A30CQAJAIAAgAFwNACAAvCIBQR92IQICQAJAAkACQAJAAkACQCABQf////8HcSIDQc/YupUESw0AIANBmOTF9QNNDQEgA0GSq5T8A00NAgwECyADQYCAgPwHSw0GAkAgAUEASA0AIANBmOTFlQRJDQAgAEMAAAB/lA8LIAFBf0oNA0MAAAAAIQQgA0G047+WBEsNBwwDCyADQYCAgMgDTQ0BQQAhA0MAAAAAIQUgACEEDAQLIAJBAXMgAmshAwwCCyAAQwAAgD+SDwsgAEM7qrg/lCACQQJ0KgKYgMCAAJL8ACEDCyAAIAOyIgRDAHIxv5SSIgAgBEOOvr81lCIFkyEECyAAIAQgBCAEIASUIgYgBkMVUjW7lEOPqio+kpSTIgaUQwAAAEAgBpOVIAWTkkMAAIA/kiEEIANFDQEgBCADEImAgIAADwsgACEECyAEC6YCAQR/AkACQCAAIABcDQAgALwiAkH/////B3FBgICA/AdPDQAgAkGAgICAeHEhA0EIIAJBAXQiBGdrIARBGHYgBEGAgIAISRshBAJAIAFBAEgNAAJAIAFB/gEgBGtMDQBDAACAfyAAmA8LAkAgBCABaiIFQQBKDQAgAiABdCADcr4PCwJAIARBAEoNACACQQEgBGt0Qf///wNxIAVBF3RyIANyvg8LIAFBF3QgAmq+DwsgAUEAIARrSg0BAkAgAUFpIARrTg0AIAO+DwtBgICABEEAIARBAEobIAJB////A3FyQQAgAWsiA0EBIAEgBGprIgEgAyABSBtBf2oiAXZBAWpBAXYgAmggAUEfcUZBf3NxviAAmCEACyAADwsgAUEXdCACar4LCykBAEGAgMAACyDNzMw9zcxMPpqZmT7NzEw+mpmZPgAAAD8AAAA/AAAAvw==";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function softmaxAt(a, b, c, temperature, index) {
+ return W.t27_softmax_at(a, b, c, temperature, int(index, 0, 4294967295, "softmax_at.index"));
+}
+function backwardAt(index) {
+ return W.t27_backward_at(int(index, 0, 4294967295, "backward_at.index"));
+}
+function weighted(a, b, c) {
+ return W.t27_weighted(a, b, c);
+}
+function gradAt(a, b, c, index) {
+ return W.t27_grad_at(a, b, c, int(index, 0, 4294967295, "grad_at.index"));
+}
+function validInput() {
+ return W.t27_valid_input();
+}
+const __NOT_LOWERED__ = Object.freeze(["forward", "backward", "cfg"]);
+export {
+ __NOT_LOWERED__,
+ backwardAt,
+ gradAt,
+ softmaxAt,
+ validInput,
+ weighted
+};
diff --git a/apps/website/public/widgets/nn-softmax/logic.t27 b/apps/website/public/widgets/nn-softmax/logic.t27
new file mode 100644
index 0000000000..859f9db7ed
--- /dev/null
+++ b/apps/website/public/widgets/nn-softmax/logic.t27
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Control sharpness of distribution | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: written with `range` and `math::exp`, which no spec declares, so no test
+// ever ran. backward was also wrong: it used only the Jacobian's diagonal,
+// p_i (1 - p_i), and dropped the -p_i p_j terms. It is now the full product
+// grad_input_i = p_i (g_i - sum_j g_j p_j) / temperature.
+
+module Softmax;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const ZERO : f32 = 0.0;
+ const ONE : f32 = 1.0;
+ const MIN_TEMP : f32 = 0.001;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const SoftmaxConfig = struct {
+ temperature : f32,
+ axis : u32,
+ stable : bool,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // output_i = exp((x_i - max) / T) / sum_j exp((x_j - max) / T); subtracting the max keeps exp finite
+ fn forward(logits: []const f32, output: []f32, config: SoftmaxConfig) -> void {
+ var max_val : f32 = logits[0];
+ var i : usize = 0;
+ while (i < logits.len) {
+ if (logits[i] > max_val) {
+ max_val = logits[i];
+ }
+ i = i + 1;
+ }
+ var sum : f32 = ZERO;
+ i = 0;
+ while (i < logits.len) {
+ output[i] = exp((logits[i] - max_val) / config.temperature);
+ sum = sum + output[i];
+ i = i + 1;
+ }
+ i = 0;
+ while (i < logits.len) {
+ output[i] = output[i] / sum;
+ i = i + 1;
+ }
+ }
+
+ // Vector-Jacobian product: grad_input_i = p_i (g_i - sum_j g_j p_j) / T
+ fn backward(grad_output: []const f32, probabilities: []const f32, grad_input: []f32, config: SoftmaxConfig) -> void {
+ var dot : f32 = ZERO;
+ var i : usize = 0;
+ while (i < grad_output.len) {
+ dot = dot + grad_output[i] * probabilities[i];
+ i = i + 1;
+ }
+ i = 0;
+ while (i < grad_output.len) {
+ grad_input[i] = probabilities[i] * (grad_output[i] - dot) / config.temperature;
+ i = i + 1;
+ }
+ }
+
+ fn cfg(temperature: f32) -> SoftmaxConfig {
+ return SoftmaxConfig{ .temperature = temperature, .axis = 0, .stable = true };
+ }
+
+ // output[index] of softmax over (a, b, c)
+ fn softmax_at(a: f32, b: f32, c: f32, temperature: f32, index: usize) -> f32 {
+ const logits : [3]f32 = [a, b, c];
+ var output : [3]f32 = [0.0] ** 3;
+ forward(&logits, &output, cfg(temperature));
+ return output[index];
+ }
+
+ // grad_input[index] for grad_output (0.1, 0.2, 0.3) at probabilities (0.2, 0.3, 0.5)
+ fn backward_at(index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ const probabilities : [3]f32 = [0.2, 0.3, 0.5];
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ // L = sum_i w_i softmax(x)_i with w = (0.1, 0.2, 0.3): the loss whose gradient backward computes
+ fn weighted(a: f32, b: f32, c: f32) -> f32 {
+ return 0.1 * softmax_at(a, b, c, 1.0, 0) + 0.2 * softmax_at(a, b, c, 1.0, 1) + 0.3 * softmax_at(a, b, c, 1.0, 2);
+ }
+
+ fn grad_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
+ const grad_output : [3]f32 = [0.1, 0.2, 0.3];
+ var probabilities : [3]f32 = [0.0] ** 3;
+ const logits : [3]f32 = [a, b, c];
+ forward(&logits, &probabilities, cfg(1.0));
+ var grad_input : [3]f32 = [0.0] ** 3;
+ backward(&grad_output, &probabilities, &grad_input, cfg(1.0));
+ return grad_input[index];
+ }
+
+ fn valid_input() -> f32 {
+ return 3.0;
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // softmax(1, 2, 3) = (0.0900306, 0.2447285, 0.6652410)
+ test "forward_basic_case"
+ given p0 = softmax_at(1.0, 2.0, 3.0, 1.0, 0)
+ when p2 = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ then p0 > 0.0 and softmax_at(1.0, 2.0, 3.0, 1.0, 1) > 0.0 and p2 > 0.0 and @abs(p0 + softmax_at(1.0, 2.0, 3.0, 1.0, 1) + p2 - 1.0) < 0.0001 and @abs(p0 - 0.0900306) < 0.000001 and @abs(p2 - 0.6652410) < 0.000001
+
+ // Subtracting the max: logits of 1000 do not overflow exp
+ test "forward_large_logits_stay_finite"
+ given p = softmax_at(1000.0, 1000.0, 1000.0, 1.0, 1)
+ then @abs(p - 1.0 / 3.0) < 0.000001
+
+ // A higher temperature flattens the distribution toward uniform
+ test "forward_temperature_flattens"
+ given sharp = softmax_at(1.0, 2.0, 3.0, 1.0, 2)
+ when flat = softmax_at(1.0, 2.0, 3.0, 10.0, 2)
+ then flat < sharp and flat > 1.0 / 3.0
+
+ // dot = 0.23: grad = (0.2 (0.1 - 0.23), 0.3 (0.2 - 0.23), 0.5 (0.3 - 0.23)) = (-0.026, -0.009, 0.035)
+ test "backward_basic_case"
+ given g0 = backward_at(0)
+ when g2 = backward_at(2)
+ then g0 != 0.0 and backward_at(1) != 0.0 and g2 != 0.0 and @abs(g0 - (-0.026)) < 0.000001 and @abs(backward_at(1) - (-0.009)) < 0.000001 and @abs(g2 - 0.035) < 0.000001
+
+ // The gradient of a function of probabilities that sum to 1 sums to 0
+ test "backward_sums_to_zero"
+ given total = backward_at(0) + backward_at(1) + backward_at(2)
+ then @abs(total) < 0.0000001
+
+ test "backward_matches_finite_difference"
+ given slope = (weighted(1.0, 2.0 + 0.001, 3.0) - weighted(1.0, 2.0 - 0.001, 3.0)) / 0.002
+ when result = grad_at(1.0, 2.0, 3.0, 1)
+ then @abs(result - slope) < 0.0005
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // logits.size() == output.size(): the first and last element are both probabilities
+ invariant softmax_constraint_0
+ assert softmax_at(1.0, 2.0, valid_input(), 1.0, 0) >= 0.0 and softmax_at(1.0, 2.0, valid_input(), 1.0, 2) >= 0.0
+
+ // config.temperature >= MIN_TEMP
+ invariant softmax_constraint_1
+ assert cfg(1.0).temperature >= MIN_TEMP
+
+ // sum(output) = 1.0 within floating precision
+ invariant softmax_constraint_2
+ assert @abs(softmax_at(1.0, 2.0, valid_input(), 1.0, 0) + softmax_at(1.0, 2.0, valid_input(), 1.0, 1) + softmax_at(1.0, 2.0, valid_input(), 1.0, 2) - 1.0) < 0.0001
diff --git a/apps/website/public/widgets/nn-softmax/nn-softmax.t27 b/apps/website/public/widgets/nn-softmax/nn-softmax.t27
new file mode 100644
index 0000000000..09d52e7cde
--- /dev/null
+++ b/apps/website/public/widgets/nn-softmax/nn-softmax.t27
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-softmax.t27 -- softmax and temperature: scores into a choice
+; Source of truth for public/widgets/nn-softmax/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-softmax/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/softmax.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 7 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets three scores from -3 to 3 and a temperature from 0.1 to 5. The bars
+; are softmax_at(a, b, c, T, i) and the sweep draws the same fn over T, with a cursor to drag.
+
+; WHAT IT DOES NOT CLAIM: three scores stand for a layer of any width; the spec's softmax_at takes
+; three.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_softmax;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-softmax";
+pub const TITLE : str = "Softmax: how a network turns scores into a choice";
+pub const DESCRIPTION : str = "Move three scores and the temperature: softmax turns them into probabilities that sum to one. Cool it and the top score takes almost everything; heat it and the choice evens out.";
+pub const IMAGE_ALT : str = "Softmax card: scores 1, 2 and 3 at temperature 1. Three bars in cyan, pink and green stand at 0.0900, 0.2447 and 0.6652. Below, three curves sweep the temperature from 0.1 to 5: the green one falls from 1 toward 0.4 while the other two rise, with a cursor at T = 1.";
+pub const HOOK : str = "Scores 1, 2, 3 give 0.090, 0.245, 0.665. At temperature 0.5 the top one takes 0.867; at 5 the three are 0.27, 0.33, 0.40.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/softmax.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 7 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SCORE_LO_C : i16 = -300;
+pub const K_SCORE_HI_C : i16 = 300;
+pub const K_T_LO_C : u8 = 10;
+pub const K_T_HI_C : u16 = 500;
+pub const K_T_STEP_C : u8 = 5;
+pub const K_T_TICKS : [6]u8 = [0, 1, 2, 3, 4, 5];
+pub const K_P_TICKS_C : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_A_C : [4]i16 = [100, 100, 100, 200];
+pub const K_PRESET_B_C : [4]i16 = [200, 200, 200, 200];
+pub const K_PRESET_C_C : [4]i16 = [300, 300, 300, 0];
+pub const K_PRESET_T_C : [4]u16 = [100, 50, 500, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["1, 2, 3 at T = 1", "1, 2, 3 at T = 0.5", "1, 2, 3 at T = 5", "A tie: 2, 2, 0"];
+pub const SAY_ASK : str = "How does a network turn scores into a choice?";
+pub const SAY_A : str = "A";
+pub const SAY_B : str = "B";
+pub const SAY_C : str = "C";
+pub const SAY_T : str = "temperature T";
+pub const SAY_BARS_LABEL : str = "The three probabilities softmax gives the three scores";
+pub const SAY_SWEEP_LABEL : str = "Each probability as the temperature goes from 0.1 to 5";
+pub const SAY_SWEEP : str = "Drag along the temperature: the same three scores, cooled and heated.";
+pub const SAY_P : str = "p({0})";
+pub const SAY_COLD : str = "at T = {0}";
+pub const SAY_NOTE : str = "Softmax is exp(x / T) over the sum of all of them, with the largest score subtracted first so exp never overflows. A language model samples its next token from exactly this.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/softmax.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_temperature_stays_positive {
+ assert K_T_LO_C > 0;
+ assert K_T_HI_C == 500;
+ assert K_PRESET_T_C[1] < K_PRESET_T_C[0];
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_C_C[0] == K_SCORE_HI_C;
+ assert K_PRESET_A_C[3] == K_PRESET_B_C[3];
+ assert K_PRESET_T_C[2] == K_T_HI_C;
+}
diff --git a/apps/website/public/widgets/nn-softmax/tool.js b/apps/website/public/widgets/nn-softmax/tool.js
new file mode 100644
index 0000000000..852840f5d9
--- /dev/null
+++ b/apps/website/public/widgets/nn-softmax/tool.js
@@ -0,0 +1,41 @@
+// nn-softmax/tool.js -- lesson 7 of neural-networks: how a network turns scores into a choice.
+// Softmax makes three scores into three probabilities that sum to one, and the temperature decides
+// how sure it is. Every bar and every point of the temperature sweep is softmax_at of gHashTag/t27
+// specs/ml/activation/softmax.t27, compiled to wasm (./logic.js). Words are SAY_ and numbers K_
+// constants of specs/widgets/nn-softmax.t27.
+import { softmaxAt } from './logic.js'
+import { COLORS, K, W, ask, bars, canvas, centi, controls, cursor, dragX, el, fill, fmt, legend, plot, root, sample, say, source, tiles } from '../nn-kit.js'
+
+const S0 = centi(K('K_SCORE_LO_C', -300)), S1 = centi(K('K_SCORE_HI_C', 300))
+const T0 = centi(K('K_T_LO_C', 10)), T1 = centi(K('K_T_HI_C', 500))
+const score = (key, label) => ({ key, label, min: S0, max: S1, step: 0.1, fmt: (v) => fmt(v, 1) })
+const ctl = controls([
+ score('a', say('SAY_A')), score('b', say('SAY_B')), score('c', say('SAY_C')),
+ { key: 't', label: say('SAY_T'), min: T0, max: T1, step: centi(K('K_T_STEP_C', 5)), fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { a: K('K_PRESET_A_C', []).map(centi), b: K('K_PRESET_B_C', []).map(centi), c: K('K_PRESET_C_C', []).map(centi), t: K('K_PRESET_T_C', []).map(centi) } }, draw)
+
+const chart = canvas('nn-bars', say('SAY_BARS_LABEL'))
+const sweep = canvas('nn-curve', say('SAY_SWEEP_LABEL'))
+const stat = el('div')
+const names = [say('SAY_A'), say('SAY_B'), say('SAY_C')]
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, chart, stat, el('p', { class: 'dk-note' }, say('SAY_SWEEP')), sweep, legend(names.map((n, i) => [n, COLORS[[2, 3, 0][i]]])), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ const p3 = [0, 1, 2].map((i) => softmaxAt(s.a, s.b, s.c, s.t, i))
+ const colors = [COLORS[2], COLORS[3], COLORS[0]]
+ bars(chart, { values: p3, labels: names.map((n, i) => `${n} = ${fmt([s.a, s.b, s.c][i], 1)}`), colors, lo: 0, hi: 1, h: 240, ticks: K('K_P_TICKS_C', []).map(centi), fmt: (v) => fmt(v, 4) })
+ sweep.replaceChildren()
+ const o = { w: 640, h: 280, x0: T0, x1: T1, y0: 0, y1: 1, xticks: K('K_T_TICKS', []), yticks: K('K_P_TICKS_C', []).map(centi), xlabel: say('SAY_T') }
+ const p = plot(sweep, o)
+ for (let i = 0; i < 3; i++) p.line(sample((t) => softmaxAt(s.a, s.b, s.c, t, i), T0, T1, 120), colors[i], 3)
+ cursor(sweep, p, s.t, p3.map((v, i) => [v, colors[i]]))
+ dragX(sweep, p, o, (t) => ctl.set('t', t))
+ stat.replaceChildren(tiles([
+ { label: fill(say('SAY_P'), names[0]), value: fmt(p3[0], 4), tone: 2 },
+ { label: fill(say('SAY_P'), names[1]), value: fmt(p3[1], 4), tone: 3 },
+ { label: fill(say('SAY_P'), names[2]), value: fmt(p3[2], 4), tone: 0 },
+ { label: fill(say('SAY_COLD'), fmt(T0, 2)), value: [0, 1, 2].map((i) => fmt(softmaxAt(s.a, s.b, s.c, T0, i), 3)).join(' '), tone: 1 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-tanh/card.png b/apps/website/public/widgets/nn-tanh/card.png
new file mode 100644
index 0000000000..952cf89b66
Binary files /dev/null and b/apps/website/public/widgets/nn-tanh/card.png differ
diff --git a/apps/website/public/widgets/nn-tanh/index.html b/apps/website/public/widgets/nn-tanh/index.html
new file mode 100644
index 0000000000..387ce94d65
--- /dev/null
+++ b/apps/website/public/widgets/nn-tanh/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+tanh: where a neuron stops learning · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
tanh: where a neuron stops learning
+
Drag along tanh: it is centred on zero, and where it flattens out its slope drops under 0.1. The red band shows every input where the gradient has all but stopped.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-tanh/logic.js b/apps/website/public/widgets/nn-tanh/logic.js
new file mode 100644
index 0000000000..b25c8855fe
--- /dev/null
+++ b/apps/website/public/widgets/nn-tanh/logic.js
@@ -0,0 +1,36 @@
+// Generated by scripts/t27-logic.mjs from tanh_activation.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 389394efbb10c82af69f052a9e7b3a929f09f45a7976930c79aee2352da4cf31
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-tanh/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABEQNgAX8BfWABfQF9YAJ9fwF9AwgHAAEBAQEBAgQFAXABAQEFAwEAEQYJAX8BQYCAwAALB1UFBm1lbW9yeQIADHQyN19iYXRjaF9hdAAAGnQyN19kZXJpdmF0aXZlX2Zyb21fb3V0cHV0AAIOdDI3X2Rlcml2YXRpdmUAAwt0MjdfZm9yd2FyZAAECvMGB3YCAn8BfSOAgICAAEEQayIBJICAgIAAQQAhAiABQQhqQQA2AgAgAUIANwMAAkADQCACQQxGDQEgASACaiACQYCAwIAAaioCABCBgICAADgCACACQQRqIQIMAAsLIAEgAEECdGoqAgAhAyABQRBqJICAgIAAIAMLTQACQCAAQwAAAABdRQ0AQwAAgD8gACAAkhCFgICAACIAk4wgAEMAAIA/kpUPC0MAAIA/IABDAAAAwJQQhYCAgAAiAJMgAEMAAIA/kpULDQBDAACAPyAAIACUkwsVAEMAAIA/IAAQgYCAgAAiACAAlJMLCgAgABCBgICAAAvUAgIDfwN9AkACQCAAIABcDQAgALwiAUEfdiECAkACQAJAAkACQAJAAkAgAUH/////B3EiA0HP2LqVBEsNACADQZjkxfUDTQ0BIANBkquU/ANNDQIMBAsgA0GAgID8B0sNBgJAIAFBAEgNACADQZjkxZUESQ0AIABDAAAAf5QPCyABQX9KDQNDAAAAACEEIANBtOO/lgRLDQcMAwsgA0GAgIDIA00NAUEAIQNDAAAAACEFIAAhBAwECyACQQFzIAJrIQMMAgsgAEMAAIA/kg8LIABDO6q4P5QgAkECdCoCjIDAgACS/AAhAwsgACADsiIEQwByMb+UkiIAIARDjr6/NZQiBZMhBAsgACAEIAQgBCAElCIGIAZDFVI1u5RDj6oqPpKUkyIGlEMAAABAIAaTlSAFk5JDAACAP5IhBCADRQ0BIAQgAxCGgICAAA8LIAAhBAsgBAumAgEEfwJAAkAgACAAXA0AIAC8IgJB/////wdxQYCAgPwHTw0AIAJBgICAgHhxIQNBCCACQQF0IgRnayAEQRh2IARBgICACEkbIQQCQCABQQBIDQACQCABQf4BIARrTA0AQwAAgH8gAJgPCwJAIAQgAWoiBUEASg0AIAIgAXQgA3K+DwsCQCAEQQBKDQAgAkEBIARrdEH///8DcSAFQRd0ciADcr4PCyABQRd0IAJqvg8LIAFBACAEa0oNAQJAIAFBaSAEa04NACADvg8LQYCAgARBACAEQQBKGyACQf///wNxckEAIAFrIgNBASABIARqayIBIAMgAUgbQX9qIgF2QQFqQQF2IAJoIAFBH3FGQX9zcb4gAJghAAsgAA8LIAFBF3QgAmq+CwsdAQBBgIDAAAsUAAAAAAAAgD8AAABAAAAAPwAAAL8=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function forward(x) {
+ return W.t27_forward(x);
+}
+function derivative(x) {
+ return W.t27_derivative(x);
+}
+function derivativeFromOutput(tanhX) {
+ return W.t27_derivative_from_output(tanhX);
+}
+function batchAt(index) {
+ return W.t27_batch_at(int(index, 0, 4294967295, "batch_at.index"));
+}
+const __NOT_LOWERED__ = Object.freeze(["forward_batch"]);
+export {
+ __NOT_LOWERED__,
+ batchAt,
+ derivative,
+ derivativeFromOutput,
+ forward
+};
diff --git a/apps/website/public/widgets/nn-tanh/logic.t27 b/apps/website/public/widgets/nn-tanh/logic.t27
new file mode 100644
index 0000000000..b6044c349c
--- /dev/null
+++ b/apps/website/public/widgets/nn-tanh/logic.t27
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// Derivative (can compute from output) | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: `math::exp` named a module this spec never imports, and
+// forward_batch allocated with `[]f32{input.len}`. tanh is now computed in the
+// form that cannot overflow: exp(2|x|) reaches inf in f32 at |x| = 45, and
+// inf / inf is NaN.
+
+module Tanh;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const TanhConfig = struct {
+ none : void,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // tanh(x) = (1 - e^(-2|x|)) / (1 + e^(-2|x|)), with the sign of x
+ fn forward(x: f32) -> f32 {
+ if (x < 0.0) {
+ return -forward(-x);
+ }
+ const t = exp(-2.0 * x);
+ return (1.0 - t) / (1.0 + t);
+ }
+
+ // forward_batch: output[i] = tanh(input[i])
+ fn forward_batch(input: []const f32, output: []f32) -> void {
+ var i : usize = 0;
+ while (i < input.len) {
+ output[i] = forward(input[i]);
+ i = i + 1;
+ }
+ }
+
+ // tanh'(x) = 1 - tanh(x)^2
+ fn derivative(x: f32) -> f32 {
+ const tanh_x = forward(x);
+ return 1.0 - (tanh_x * tanh_x);
+ }
+
+ // Given y = tanh(x), the derivative is 1 - y^2
+ fn derivative_from_output(tanh_x: f32) -> f32 {
+ return 1.0 - (tanh_x * tanh_x);
+ }
+
+ fn batch_at(index: usize) -> f32 {
+ const input : [3]f32 = [0.0, 1.0, 2.0];
+ var output : [3]f32 = [0.0] ** 3;
+ forward_batch(&input, &output);
+ return output[index];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // tanh(0) = 0, tanh(1) = 0.7615942, tanh is odd
+ test forward_basic_case
+ given input = 0.0
+ when result = forward(input)
+ then result == 0.0 and @abs(forward(1.0) - 0.7615942) < 0.000001 and forward(-1.0) == -forward(1.0)
+
+ // exp(2x) is inf in f32 here; the stable form still gives 1, not NaN
+ test forward_saturates_without_overflow
+ given input = 100.0
+ when result = forward(input)
+ then result == 1.0 and forward(-100.0) == -1.0
+
+ test forward_batch_basic_case
+ given first = batch_at(0)
+ when second = batch_at(1)
+ then first == 0.0 and second > 0.0 and second < 1.0 and batch_at(2) > second
+
+ test derivative_basic_case
+ given input = 0.0
+ when result = derivative(input)
+ then result == 1.0
+
+ test derivative_from_output_basic_case
+ given input = 0.5
+ when tanh_result = forward(input)
+ then derivative_from_output(tanh_result) > 0.0 and derivative_from_output(tanh_result) < 1.0 and derivative_from_output(tanh_result) == derivative(input)
+
+ test derivative_matches_finite_difference
+ given slope = (forward(0.5 + 0.001) - forward(0.5 - 0.001)) / 0.002
+ when result = derivative(0.5)
+ then @abs(result - slope) < 0.001
diff --git a/apps/website/public/widgets/nn-tanh/nn-tanh.t27 b/apps/website/public/widgets/nn-tanh/nn-tanh.t27
new file mode 100644
index 0000000000..e842bc0f11
--- /dev/null
+++ b/apps/website/public/widgets/nn-tanh/nn-tanh.t27
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-tanh.t27 -- tanh and the saturated band where learning stops
+; Source of truth for public/widgets/nn-tanh/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-tanh/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/tanh_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 3 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -4 to 4. The curve is forward, the slope derivative, and
+; every sampled x whose slope is under K_SAT is shaded red.
+
+; WHAT IT DOES NOT CLAIM: the threshold 0.1 is a choice made for the picture, not a constant of the
+; spec.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_tanh;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-tanh";
+pub const TITLE : str = "tanh: where a neuron stops learning";
+pub const DESCRIPTION : str = "Drag along tanh: it is centred on zero, and where it flattens out its slope drops under 0.1. The red band shows every input where the gradient has all but stopped.";
+pub const IMAGE_ALT : str = "tanh card at x = 2.5: the green curve from -1 to 1, the dashed gold slope a bell peaking at 1, and two red bands on both sides where the slope is under 0.1. Tiles read tanh 0.9866, slope 0.0266 marked saturated, and the spec batch tanh(0, 1, 2) = 0 0.762 0.964.";
+pub const HOOK : str = "tanh(2.5) is already 0.987 and its slope 0.027. Past about 1.8 either way the gradient is under 0.1: the band a deep net must stay out of.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/tanh_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 3 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -400;
+pub const K_X_HI_C : i16 = 400;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [3]i8 = [-1, 0, 1];
+pub const K_SAT_C : u8 = 10;
+pub const K_PRESET_X_C : [4]i16 = [250, 0, 100, -300];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["x = 2.5", "x = 0", "x = 1", "x = -3"];
+pub const SAY_ASK : str = "Where does a neuron stop learning?";
+pub const SAY_X : str = "input x";
+pub const SAY_F : str = "tanh(x)";
+pub const SAY_DF : str = "slope";
+pub const SAY_SAT_BAND : str = "slope under 0.1";
+pub const SAY_PLOT_LABEL : str = "tanh of x and its slope from -4 to 4, saturated inputs shaded";
+pub const SAY_OUT : str = "tanh(x)";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_BATCH : str = "spec batch 0 1 2";
+pub const SAY_STATE : str = "here";
+pub const SAY_SATURATED : str = "saturated";
+pub const SAY_LEARNING : str = "still learning";
+pub const SAY_NOTE : str = "Unlike the sigmoid, tanh is centred on 0 and its slope reaches 1. It still saturates: the gradient that flows back through it is multiplied by that slope.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/tanh_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_band_threshold_is_a_tenth {
+ assert K_SAT_C == 10;
+ assert K_SAT_C < 100;
+}
+
+test the_presets_cover_both_regions {
+ assert K_PRESET_X_C[0] > 150;
+ assert K_PRESET_X_C[1] == 0;
+ assert K_PRESET_X_C[3] + 150 < 0;
+}
diff --git a/apps/website/public/widgets/nn-tanh/tool.js b/apps/website/public/widgets/nn-tanh/tool.js
new file mode 100644
index 0000000000..744473441f
--- /dev/null
+++ b/apps/website/public/widgets/nn-tanh/tool.js
@@ -0,0 +1,40 @@
+// nn-tanh/tool.js -- lesson 3 of neural-networks: where a neuron stops learning. tanh centres the
+// output on zero, and where it saturates its slope, and the gradient that flows back through it,
+// falls toward zero. The curve, the slope and the shaded saturated band are forward and derivative
+// of gHashTag/t27 specs/ml/activation/tanh_activation.t27, compiled to wasm (./logic.js). Words are
+// SAY_ and numbers K_ constants of specs/widgets/nn-tanh.t27.
+import { batchAt, derivative, forward } from './logic.js'
+import { COLORS, K, W, ask, canvas, centi, controls, cursor, dragX, el, fmt, legend, plot, root, sample, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const X0 = centi(K('K_X_LO_C', -400)), X1 = centi(K('K_X_HI_C', 400))
+const SAT = centi(K('K_SAT_C', 10))
+const ctl = controls([
+ { key: 'x', label: say('SAY_X'), min: X0, max: X1, step: centi(K('K_X_STEP_C', 5)), fmt: (v) => fmt(v, 2) },
+], { names: W.SAY_PRESET_NAMES, values: { x: K('K_PRESET_X_C', []).map(centi) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_PLOT_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, stat, svg, legend([[say('SAY_F'), COLORS[0]], [say('SAY_DF'), COLORS[1], true], [say('SAY_SAT_BAND'), COLORS[6]]]), el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+function draw(s) {
+ svg.replaceChildren()
+ const o = { w: 640, h: 330, x0: X0, x1: X1, y0: -1.1, y1: 1.1, xticks: K('K_X_TICKS', []), yticks: K('K_Y_TICKS', []), xlabel: say('SAY_X') }
+ const p = plot(svg, o)
+ // The saturated band: every sampled x whose slope is under K_SAT, shaded.
+ const pts = sample((x) => derivative(x), X0, X1, 200)
+ const dx = (X1 - X0) / 200
+ for (const [x, d] of pts) if (d < SAT) svg.append(svgEl('rect', { x: p.x(x - dx / 2), y: p.top, width: Math.max(1, p.x(x + dx / 2) - p.x(x - dx / 2)), height: p.clampY(-Infinity) - p.top, fill: COLORS[6], opacity: 0.12 }))
+ p.line(sample((x) => forward(x), X0, X1), COLORS[0], 3)
+ p.line(pts, COLORS[1], 2, { 'stroke-dasharray': '6 5' })
+ cursor(svg, p, s.x, [[forward(s.x), COLORS[0]], [derivative(s.x), COLORS[1]]])
+ dragX(svg, p, o, (x) => ctl.set('x', x))
+ const d = derivative(s.x)
+ stat.replaceChildren(tiles([
+ { label: say('SAY_OUT'), value: fmt(forward(s.x), 4), tone: 0 },
+ { label: say('SAY_GRAD'), value: fmt(d, 4), tone: d < SAT ? 'bad' : 1 },
+ { label: say('SAY_BATCH'), value: [0, 1, 2].map((i) => fmt(batchAt(i), 3)).join(' '), tone: 2 },
+ { label: say('SAY_STATE'), value: d < SAT ? say('SAY_SATURATED') : say('SAY_LEARNING'), tone: d < SAT ? 'bad' : 0 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/public/widgets/nn-xor/card.png b/apps/website/public/widgets/nn-xor/card.png
new file mode 100644
index 0000000000..f25925aee0
Binary files /dev/null and b/apps/website/public/widgets/nn-xor/card.png differ
diff --git a/apps/website/public/widgets/nn-xor/index.html b/apps/website/public/widgets/nn-xor/index.html
new file mode 100644
index 0000000000..6b30f34899
--- /dev/null
+++ b/apps/website/public/widgets/nn-xor/index.html
@@ -0,0 +1,47 @@
+
+
+
+
+
+
+XOR: a network learns what one neuron cannot · Trinity S³AI
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Trinity S³AI
+
XOR: a network learns what one neuron cannot
+
A 2-8-1 network trained by backpropagation in your browser: drag the epochs from 0 to 2000 and watch its four answers move to 0, 1, 1, 0 while the loss falls from 0.89 by twelve orders.
+
+
+
+
+
+
+
+
diff --git a/apps/website/public/widgets/nn-xor/logic.js b/apps/website/public/widgets/nn-xor/logic.js
new file mode 100644
index 0000000000..f0039ed891
--- /dev/null
+++ b/apps/website/public/widgets/nn-xor/logic.js
@@ -0,0 +1,32 @@
+// Generated by scripts/t27-logic.mjs from mlp.t27. Do not edit.
+//
+// The spec's own fn bodies, compiled: t27c gen (Zig) -> zig 0.16.0 (wasm32).
+// Change the spec in gHashTag/t27, vendor it, regenerate. A rule written
+// by hand beside this file is the thing this file exists to end.
+// spec-sha256: 936f35f57c99851545afe0b1797b416777549610a3f57cf49c0727311eef281a
+// Imported by scripts/widget-logic.mjs for public/widgets/nn-xor/: types stripped, nothing else changed.
+const WASM = "AGFzbQEAAAABMAdgAn9/AX1gAn9/AGAFf39/f38AYAR/f39/AGAHf39/f39/fwBgAX8BfWACfX0BfQMLCgABAgMBBAUABgYEBQFwAQEBBQMBABcGCQF/AUGAgMAACwc5BAZtZW1vcnkCAAd0MjdfeG9yAAAOdDI3X21uaXN0X2luaXQABhF0MjdfbW5pc3RfZm9yd2FyZAAHCoYRCs8FBwJ/An0CfwF9AX8BfQR/I4CAgIAAQfABayICJICAgIAAIAJBATYCqAEgAkEINgKgASACQQg2ApgBIAJBEDYCkAEgAiACQYgBajYCpAEgAiACQegAajYCnAEgAiACQcgAajYClAEgAiACQQhqNgKMASACQYwBakGAgMCAABCBgICAAEEAIQNDAAAAACEEQwAAAAAhBQJAA0AgAyAARg0BQwAAAAAhBEEAIQZBjIDAgAAhBwJAA0AgBkEERg0BIAZBA3RBjIDAgABqIAJBjAFqIAJBrAFqIAJB7AFqQYCAwIAAEIKAgIAAIAIqAuwBIAZBAnQqAqyAwIAAkyEIQQAhCQJAA0AgCUEgRg0BQwAAAAAhCgJAIAJBrAFqIAlqKgIAQwAAAABeRQ0AIAggAkHoAGogCWoqAgCUQwAAAACSIQoLIAJBzAFqIAlqIAo4AgAgCUEEaiEJDAALCyAIQ83MTD2UIQpBACEJAkADQCAJQSBGDQEgAkHoAGogCWoiCyALKgIAIAogAkGsAWogCWoqAgCUkzgCACAJQQRqIQkMAAsLIAIgAioCiAEgCpM4AogBQQAhDCACQQhqIQ0CQANAIAxBCEYNASACQcwBaiAMQQJ0Ig5qKgIAQ83MTD2UIQpBACEJAkADQCAJQQhGDQEgDSAJaiILIAsqAgAgCiAHIAlqKgIAlJM4AgAgCUEEaiEJDAALCyACQcgAaiAOaiIJIAkqAgAgCpM4AgAgDUEIaiENIAxBAWohDAwACwsgB0EIaiEHIAZBAWohBiAEIAggCEMAAAA/lJRDAAAAAJKSIQQMAAsLIAUgBCADGyEFIANBAWohAwwACwsCQAJAAkAgAQ4CAgEACyABQQN0Qfz/v4AAaiACQYwBaiACQawBaiACQewBakGAgMCAABCCgICAACACKgLsASEFDAELIAQhBQsgAkHwAWokgICAgAAgBQtuAQd/IAAoAhwhAiAAKAIYIQMgASgCBCEEIAAoAhQhBSAAKAIQIQYgACgCDCEHIAAoAgghCCAAKAIAIAAoAgQgASgCAEEBEIOAgIAAIAggBxCEgICAACAGIAUgBEEHEIOAgIAAIAMgAhCEgICAAAtPAQN/IAQoAgghBSABKAIYIQYgASgCECEHIAAgASgCACABKAIIIAQoAgAgBCgCBCIBQQEgAhCFgICAACACIAcgBiABIAVBACADEIWAgIAAC2ABAX0gA0Gx893xeWwhA0MAAABAIAKzlZEhBAJAA0AgA0HoB3AhAiABRQ0BIAAgBCACs0MAAPpDlUMAAIC/kpQ4AgAgAEEEaiEAIANBsfPd8XlqIQMgAUF/aiEBDAALCwskAAJAA0AgAUUNASAAQQA2AgAgAEEEaiEAIAFBf2ohAQwACwsLoQEDBH8BfQJ/IANBAnQhB0EAIQggBUEBcSEJAkADQCAIIARGDQEgAiAIQQJ0IgpqKgIAIQsgACEMIAEhDSADIQUCQANAIAVFDQEgBUF/aiEFIAsgDSoCACAMKgIAlJIhCyAMQQRqIQwgDUEEaiENDAALCyAGIApqQwAAAAAgCyALQwAAAABdGyALIAkbOAIAIAEgB2ohASAIQQFqIQgMAAsLC/cCBAF/AX0BfwJ9I4CAgIAAQdDsGGsiASSAgICAAAJAQYDAGEUNACABQQhqQbyAwIAAQYDAGPwKAAALAkBBgARFDQAgAUGIwBhqQbzA2IAAQYAE/AoAAAsCQEEoRQ0AIAFBiOwYakG8xNiAAEEo/AoAAAsgAUEKNgLM7BggAUGACjYCxOwYIAFBgAE2ArzsGCABQYCQBjYCtOwYIAEgAUGI7BhqNgLI7BggASABQYjEGGo2AsDsGCABIAFBiMAYajYCuOwYIAEgAUEIajYCsOwYIAFBsOwYakHkxNiAABCBgICAAEMAAAAAIQJBACEDQwAAAAAhBAJAA0AgA0GAwBhGDQEgBEMAAIA/kiAEIAFBCGogA2oqAgAiBUMAABBBXBshBCADQQRqIQMgAiAFixCJgICAACECDAALCwJAAkACQAJAIAAOAwMAAQILIAEqAojAGCEEDAILIAEqAojsGCEEDAELIAIhBAsgAUHQ7BhqJICAgIAAIAQLxQMCA38CfSOAgICAAEHAiRlrIgIkgICAgAAgAkEKNgLQ7BggAkGACjYCyOwYIAJBgAE2AsDsGCACQYCQBjYCuOwYIAIgAkGM7BhqNgLM7BggAiACQYzEGGo2AsTsGCACIAJBjMAYajYCvOwYIAIgAkEMajYCtOwYIAJBtOwYakHkxNiAABCBgICAAEEAIQMgAkHU7BhqIQQCQANAIANBkAZGDQEgBCADQf//A3FBB3CzQwAA4ECVOAIAIARBBGohBCADQQFqIQMMAAsLAkBBLEUNACACQZSJGWpB8MTYgABBLPwKAAALIAJB1OwYaiACQbTsGGogAkGUhRlqIAJBlIkZakHkxNiAABCCgICAAAJAAkAgAA0AIAJBlIkZaiABQQJ0aioCACEFDAELQQAhAyACKgKUhRkhBgNAAkAgA0GABEcNAEMAAAAAIQVBACEDAkADQCADQSxGDQEgBUMAAIA/kiAFIAJBlIkZaiADaioCAEMAAMbCXBshBSADQQRqIQMMAAsLIAYgBSAAQQFGGyEFDAILIAYgAkGUhRlqIANqKgIAEIiAgIAAIQYgA0EEaiEDDAALCyACQcCJGWokgICAgAAgBQtFAQF/AkAgACAAXA0AAkAgASABWw0AIAAPCwJAIAG8IAC8IgJzQX9KDQAgACABIAJBAEgbDwsgACABIAAgAV0bIQELIAELRQEBfwJAIAAgAFwNAAJAIAEgAVsNACAADwsCQCABvCAAvCICc0F/Sg0AIAEgACACQQBIGw8LIAEgACAAIAFdGyEBCyABCwunxRgBAEGAgMAAC5zFGAIAAAAIAAAAAQAAAAAAAAAAAAAAAAAAAAAAgD8AAIA/AAAAAAAAgD8AAIA/AAAAAAAAgD8AAIA/AAAAAAAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBAAAQQQAAEEEAABBBEAMAAIAAAAAKAAAAAADGwgAAxsIAAMbCAADGwgAAxsIAAMbCAADGwgAAxsIAAMbCAADGwgAAxsI=";
+const W = new WebAssembly.Instance(
+ new WebAssembly.Module(Uint8Array.from(atob(WASM), (c) => c.charCodeAt(0)))
+).exports;
+function int(v, lo, hi, at) {
+ if (!Number.isInteger(v) || v < lo || v > hi)
+ throw new RangeError(`${at}: ${v} is outside ${lo}..${hi}`);
+ return v;
+}
+function mnistForward(which, index) {
+ return W.t27_mnist_forward(int(which, 0, 255, "mnist_forward.which"), int(index, 0, 4294967295, "mnist_forward.index"));
+}
+function mnistInit(which) {
+ return W.t27_mnist_init(int(which, 0, 255, "mnist_init.which"));
+}
+function xor(epochs, which) {
+ return W.t27_xor(int(epochs, 0, 4294967295, "xor.epochs"), int(which, 0, 4294967295, "xor.which"));
+}
+const __NOT_LOWERED__ = Object.freeze(["layer", "forward", "init_weights", "zero", "init", "train_step", "mnist"]);
+export {
+ __NOT_LOWERED__,
+ mnistForward,
+ mnistInit,
+ xor
+};
diff --git a/apps/website/public/widgets/nn-xor/logic.t27 b/apps/website/public/widgets/nn-xor/logic.t27
new file mode 100644
index 0000000000..df03c810ed
--- /dev/null
+++ b/apps/website/public/widgets/nn-xor/logic.t27
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: Apache-2.0
+// t27/specs/
+// ReLU activation for hidden layers | φ² + 1/φ² = 3 | TRINITY
+
+// #8980: forward computed the hidden layer and wrote it into `output` (the
+// second layer never ran); init called an undeclared `alloc_f32`; the tests
+// called undeclared `create_test_*` helpers; and the invariants asserted `true`.
+// The MLP is now two layers, h = relu(W1 x + b1) and y = W2 h + b2, over buffers
+// the caller owns. train_step adds backpropagation with an SGD update, and the
+// tests train the network on XOR, which no single linear layer can fit.
+
+module Mlp;
+ use base::types;
+ use math::constants;
+
+ // ═══════════════════════════════════════════════════════════
+ // 1. Constants
+ // ═══════════════════════════════════════════════════════════
+
+ const MNIST_INPUT_SIZE : u32 = 784;
+ const MNIST_OUTPUT_SIZE : u32 = 10;
+ const DEFAULT_HIDDEN_SIZE : u32 = 128;
+
+ // ═══════════════════════════════════════════════════════════
+ // 2. Types
+ // ═══════════════════════════════════════════════════════════
+
+ pub const MLPConfig = struct {
+ input_size : u32,
+ hidden_size : u32,
+ output_size : u32,
+ };
+
+ pub const MLPState = struct {
+ w1 : []f32,
+ b1 : []f32,
+ w2 : []f32,
+ b2 : []f32,
+ };
+
+ // ═══════════════════════════════════════════════════════════
+ // 3. Core Functions
+ // ═══════════════════════════════════════════════════════════
+
+ // One dense layer: out[o] = sum_i w[o * n_in + i] in[i] + b[o], relu when asked
+ fn layer(input: []const f32, w: []const f32, b: []const f32, n_in: u32, n_out: u32, relu: bool, out: []f32) -> void {
+ var o : usize = 0;
+ while (o < n_out) {
+ var sum : f32 = b[o];
+ var i : usize = 0;
+ while (i < n_in) {
+ sum = sum + w[o * n_in + i] * input[i];
+ i = i + 1;
+ }
+ if (relu and sum < 0.0) {
+ sum = 0.0;
+ }
+ out[o] = sum;
+ o = o + 1;
+ }
+ }
+
+ // h = relu(W1 x + b1) into `hidden`, y = W2 h + b2 into `output`
+ fn forward(input: []const f32, state: MLPState, hidden: []f32, output: []f32, config: MLPConfig) -> void {
+ layer(input, state.w1, state.b1, config.input_size, config.hidden_size, true, hidden);
+ layer(hidden, state.w2, state.b2, config.hidden_size, config.output_size, false, output);
+ }
+
+ // He-scaled deterministic init: w in [-1, 1) sqrt(2 / fan_in) from a fixed hash of the
+ // index (salt tells the layers apart); every bias 0
+ fn init_weights(w: []f32, fan_in: u32, salt: usize) -> void {
+ const scale : f32 = sqrt(2.0 / @as(f32, @floatFromInt(fan_in)));
+ var k : usize = 0;
+ while (k < w.len) {
+ const h : u32 = @as(u32, @intCast(((k + salt) * 2654435761) % 1000));
+ w[k] = (@as(f32, @floatFromInt(h)) / 500.0 - 1.0) * scale;
+ k = k + 1;
+ }
+ }
+
+ fn zero(v: []f32) -> void {
+ var k : usize = 0;
+ while (k < v.len) {
+ v[k] = 0.0;
+ k = k + 1;
+ }
+ }
+
+ fn init(state: MLPState, config: MLPConfig) -> void {
+ init_weights(state.w1, config.input_size, 1);
+ zero(state.b1);
+ init_weights(state.w2, config.hidden_size, 7);
+ zero(state.b2);
+ }
+
+ // One SGD step on the loss 0.5 * sum_o (y_o - t_o)^2; returns the loss before the step.
+ // d2_o = y_o - t_o (output error)
+ // d1_i = [z_i > 0] sum_o W2[o][i] d2_o (backpropagated through relu)
+ // W2 -= lr d2 h^T, b2 -= lr d2; W1 -= lr d1 x^T, b1 -= lr d1
+ fn train_step(input: []const f32, target: []const f32, state: MLPState, hidden: []f32, delta: []f32, output: []f32, lr: f32, config: MLPConfig) -> f32 {
+ forward(input, state, hidden, output, config);
+ var loss : f32 = 0.0;
+ var o : usize = 0;
+ while (o < config.output_size) {
+ const e = output[o] - target[o];
+ loss = loss + 0.5 * e * e;
+ o = o + 1;
+ }
+ var i : usize = 0;
+ while (i < config.hidden_size) {
+ var back : f32 = 0.0;
+ if (hidden[i] > 0.0) {
+ o = 0;
+ while (o < config.output_size) {
+ back = back + state.w2[o * config.hidden_size + i] * (output[o] - target[o]);
+ o = o + 1;
+ }
+ }
+ delta[i] = back;
+ i = i + 1;
+ }
+ o = 0;
+ while (o < config.output_size) {
+ const d2 = output[o] - target[o];
+ i = 0;
+ while (i < config.hidden_size) {
+ state.w2[o * config.hidden_size + i] = state.w2[o * config.hidden_size + i] - lr * d2 * hidden[i];
+ i = i + 1;
+ }
+ state.b2[o] = state.b2[o] - lr * d2;
+ o = o + 1;
+ }
+ i = 0;
+ while (i < config.hidden_size) {
+ var j : usize = 0;
+ while (j < config.input_size) {
+ state.w1[i * config.input_size + j] = state.w1[i * config.input_size + j] - lr * delta[i] * input[j];
+ j = j + 1;
+ }
+ state.b1[i] = state.b1[i] - lr * delta[i];
+ i = i + 1;
+ }
+ return loss;
+ }
+
+ fn mnist() -> MLPConfig {
+ return MLPConfig{ .input_size = MNIST_INPUT_SIZE, .hidden_size = DEFAULT_HIDDEN_SIZE, .output_size = MNIST_OUTPUT_SIZE };
+ }
+
+ // which: 0 = output[index], 1 = min over every output, 2 = number of outputs written,
+ // on the MNIST shape with input pixel k = (k % 7) / 7
+ fn mnist_forward(which: u8, index: usize) -> f32 {
+ var w1 : [784 * 128]f32 = undefined;
+ var b1 : [128]f32 = undefined;
+ var w2 : [128 * 10]f32 = undefined;
+ var b2 : [10]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, mnist());
+ var input : [784]f32 = undefined;
+ var k : usize = 0;
+ while (k < 784) {
+ input[k] = @as(f32, @floatFromInt(k % 7)) / 7.0;
+ k = k + 1;
+ }
+ var hidden : [128]f32 = undefined;
+ var output : [11]f32 = [-99.0] ** 11;
+ forward(&input, state, &hidden, &output, mnist());
+ if (which == 0) {
+ return output[index];
+ }
+ var min_h : f32 = hidden[0];
+ var written : f32 = 0.0;
+ k = 0;
+ while (k < 128) {
+ min_h = @min(min_h, hidden[k]);
+ k = k + 1;
+ }
+ k = 0;
+ while (k < 11) {
+ if (output[k] != -99.0) {
+ written = written + 1.0;
+ }
+ k = k + 1;
+ }
+ if (which == 1) {
+ return min_h;
+ }
+ return written;
+ }
+
+ // which: 0 = w1.len written by init, 1 = b1[0], 2 = b2[0], 3 = largest |w1|
+ fn mnist_init(which: u8) -> f32 {
+ var w1 : [784 * 128]f32 = [9.0] ** (784 * 128);
+ var b1 : [128]f32 = [9.0] ** 128;
+ var w2 : [128 * 10]f32 = [9.0] ** (128 * 10);
+ var b2 : [10]f32 = [9.0] ** 10;
+ init(MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 }, mnist());
+ var written : f32 = 0.0;
+ var largest : f32 = 0.0;
+ var k : usize = 0;
+ while (k < w1.len) {
+ if (w1[k] != 9.0) {
+ written = written + 1.0;
+ }
+ largest = @max(largest, @abs(w1[k]));
+ k = k + 1;
+ }
+ if (which == 0) {
+ return written;
+ }
+ if (which == 1) {
+ return b1[0];
+ }
+ if (which == 2) {
+ return b2[0];
+ }
+ return largest;
+ }
+
+ // Train a 2-8-1 network on XOR for `epochs` passes over the four cases.
+ // which: 0 = total loss of the first epoch, 1 = total loss of the last epoch,
+ // 2 + c = the trained network's output for case c
+ fn xor(epochs: u32, which: usize) -> f32 {
+ const cfg = MLPConfig{ .input_size = 2, .hidden_size = 8, .output_size = 1 };
+ var w1 : [16]f32 = undefined;
+ var b1 : [8]f32 = undefined;
+ var w2 : [8]f32 = undefined;
+ var b2 : [1]f32 = undefined;
+ const state = MLPState{ .w1 = &w1, .b1 = &b1, .w2 = &w2, .b2 = &b2 };
+ init(state, cfg);
+ const xs : [8]f32 = [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0];
+ const ts : [4]f32 = [0.0, 1.0, 1.0, 0.0];
+ var hidden : [8]f32 = undefined;
+ var delta : [8]f32 = undefined;
+ var output : [1]f32 = undefined;
+ var first : f32 = 0.0;
+ var last : f32 = 0.0;
+ var e : u32 = 0;
+ while (e < epochs) {
+ var total : f32 = 0.0;
+ var c : usize = 0;
+ while (c < 4) {
+ total = total + train_step(xs[c * 2 .. c * 2 + 2], ts[c .. c + 1], state, &hidden, &delta, &output, 0.05, cfg);
+ c = c + 1;
+ }
+ if (e == 0) {
+ first = total;
+ }
+ last = total;
+ e = e + 1;
+ }
+ if (which == 0) {
+ return first;
+ }
+ if (which == 1) {
+ return last;
+ }
+ forward(xs[(which - 2) * 2 .. (which - 2) * 2 + 2], state, &hidden, &output, cfg);
+ return output[0];
+ }
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Tests (from .tri behaviors)
+ // ═══════════════════════════════════════════════════════════
+
+ // 784 -> 128 -> 10: exactly the 10 outputs are written, the 11th slot is untouched
+ test forward_output_size
+ given written = mnist_forward(2, 0)
+ when spare = mnist_forward(0, 10)
+ then written == 10.0 and spare == -99.0
+
+ // The hidden layer is relu, so no hidden unit is negative
+ test forward_relu_activation
+ given min_hidden = mnist_forward(1, 0)
+ then min_hidden >= 0.0
+
+ // init writes every one of the 784 * 128 first-layer weights, each within sqrt(2 / 784) = 0.05051
+ test init_allocation_sizes
+ given written = mnist_init(0)
+ when largest = mnist_init(3)
+ then written == 100352.0 and largest <= 0.0506 and largest > 0.0
+
+ test init_bias_initialization
+ given b1 = mnist_init(1)
+ when b2 = mnist_init(2)
+ then b1 == 0.0 and b2 == 0.0
+
+ // Backpropagation learns XOR: the loss falls by more than 100x, and each output is within 0.1
+ test train_step_learns_xor
+ given first = xor(2000, 0)
+ when last = xor(2000, 1)
+ then last < first / 100.0 and @abs(xor(2000, 2)) < 0.1 and @abs(xor(2000, 3) - 1.0) < 0.1 and @abs(xor(2000, 4) - 1.0) < 0.1 and @abs(xor(2000, 5)) < 0.1
+
+ // ═══════════════════════════════════════════════════════════
+ // TDD: Invariants (from .tri constraints)
+ // ═══════════════════════════════════════════════════════════
+
+ // input_size == MNIST_INPUT_SIZE (784)
+ invariant mlp_constraint_0
+ assert mnist().input_size == 784
+
+ // output_size == MNIST_OUTPUT_SIZE (10)
+ invariant mlp_constraint_1
+ assert mnist().output_size == 10
+
+ // hidden_size > 0
+ invariant mlp_constraint_2
+ assert mnist().hidden_size > 0
+
+ // w1.size() == input_size * hidden_size
+ invariant mlp_constraint_3
+ assert mnist().input_size * mnist().hidden_size == 100352
+
+ // w2.size() == hidden_size * output_size
+ invariant mlp_constraint_4
+ assert mnist().hidden_size * mnist().output_size == 1280
+
+ // b1.size() == hidden_size
+ invariant mlp_constraint_5
+ assert DEFAULT_HIDDEN_SIZE == 128
+
+ // b2.size() == output_size
+ invariant mlp_constraint_6
+ assert MNIST_OUTPUT_SIZE == 10
diff --git a/apps/website/public/widgets/nn-xor/nn-xor.t27 b/apps/website/public/widgets/nn-xor/nn-xor.t27
new file mode 100644
index 0000000000..262c463c48
--- /dev/null
+++ b/apps/website/public/widgets/nn-xor/nn-xor.t27
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-xor.t27 -- a 2-8-1 network learns XOR by backpropagation
+; Source of truth for public/widgets/nn-xor/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-xor/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/pathway/mlp.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 20 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the epochs from 0 to 2000. Every output is xor(epochs, 2 + c), the
+; loss curve xor(e, 1) every 25 epochs and the first-epoch loss xor(e, 0): the spec trains its own
+; 2-8-1 network from fixed initial weights at lr 0.05, from scratch on every call.
+
+; WHAT IT DOES NOT CLAIM: the grid shows the four inputs the spec exposes, not a decision region
+; over the plane: the spec has no fn for a point between the corners.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_xor;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-xor";
+pub const TITLE : str = "XOR: a network learns what one neuron cannot";
+pub const DESCRIPTION : str = "A 2-8-1 network trained by backpropagation in your browser: drag the epochs from 0 to 2000 and watch its four answers move to 0, 1, 1, 0 while the loss falls from 0.89 by twelve orders.";
+pub const IMAGE_ALT : str = "XOR card after 2000 epochs: a 2 x 2 grid of the four cases, dark blue 0.000 on the diagonal and white 1.000 off it. Beside it a red loss curve on a log axis falls from 0.89 at epoch 1 to about 1e-13 at 2000. Tiles read first-epoch loss 0.8896, last 1.78e-13, outputs 0.00 1.00 1.00 0.00 against the targets 0 1 1 0.";
+pub const HOOK : str = "One neuron cannot draw XOR. Eight hidden units can: after 2000 epochs the outputs are 0, 1, 1, 0 to within 4e-7 and the loss is 1.8e-13.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/pathway/mlp.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 20 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_EPOCHS_HI : u16 = 2000;
+pub const K_CURVE_STEP : u8 = 25;
+pub const K_LOG_LO : i8 = -14;
+pub const K_LOG_HI : u8 = 1;
+pub const K_LOG_TICKS : [5]i8 = [-12, -9, -6, -3, 0];
+pub const K_EPOCH_TICKS : [5]u16 = [0, 500, 1000, 1500, 2000];
+pub const K_CASES : [4]u8 = [0, 1, 1, 0];
+pub const K_PRESET_EPOCHS : [4]u16 = [2000, 0, 100, 500];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Trained, 2000 epochs", "Untrained", "100 epochs", "500 epochs"];
+pub const SAY_ASK : str = "Can a network learn something one neuron cannot?";
+pub const SAY_EPOCHS : str = "epochs";
+pub const SAY_A : str = "a =";
+pub const SAY_B : str = "b =";
+pub const SAY_CURVE_LABEL : str = "The training loss, log scale";
+pub const SAY_GRID_LABEL : str = "The network's answer for each input (a, b)";
+pub const SAY_LOG_NOTE : str = "log scale";
+pub const SAY_FIRST_LOSS : str = "loss, epoch 1";
+pub const SAY_LAST_LOSS : str = "loss, last epoch";
+pub const SAY_UNTRAINED : str = "not trained";
+pub const SAY_OUTPUTS : str = "outputs 00 01 10 11";
+pub const SAY_TARGETS : str = "targets";
+pub const SAY_NOTE : str = "XOR is 1 when exactly one input is 1. No straight line separates its 1s from its 0s, so one neuron fails; a hidden layer of ReLUs bends the space until one does.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/pathway/mlp.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_curve_steps_divide_the_run {
+ assert K_EPOCHS_HI % K_CURVE_STEP == 0;
+ assert K_EPOCHS_HI / K_CURVE_STEP == 80;
+}
+
+test the_targets_are_xor {
+ assert K_CASES[0] == 0;
+ assert K_CASES[1] == 1;
+ assert K_CASES[2] == 1;
+ assert K_CASES[3] == 0;
+ assert K_PRESET_EPOCHS[0] == K_EPOCHS_HI;
+}
diff --git a/apps/website/public/widgets/nn-xor/tool.js b/apps/website/public/widgets/nn-xor/tool.js
new file mode 100644
index 0000000000..af4781e3c8
--- /dev/null
+++ b/apps/website/public/widgets/nn-xor/tool.js
@@ -0,0 +1,46 @@
+// nn-xor/tool.js -- lesson 20 of neural-networks: can a network learn what one neuron cannot? XOR
+// is not linearly separable, yet a 2-8-1 network trained by backpropagation gets all four cases
+// right. Every output and every point of the loss curve is xor(epochs, which) of gHashTag/t27
+// specs/ml/pathway/mlp.t27, compiled to wasm (./logic.js): the spec trains from its own fixed
+// initial weights, lr 0.05, each call from scratch. The four squares are the four inputs the spec
+// exposes, not the whole plane. Words are SAY_ and numbers K_ constants of specs/widgets/nn-xor.t27.
+import { xor } from './logic.js'
+import { COLORS, K, W, ask, canvas, controls, cursor, dragX, el, fmt, heat, heatTone, plot, root, say, source, svgEl, tiles } from '../nn-kit.js'
+
+const EMAX = K('K_EPOCHS_HI', 2000), STEP = K('K_CURVE_STEP', 25)
+const LOG0 = K('K_LOG_LO', -14), LOG1 = K('K_LOG_HI', 1)
+const ctl = controls([
+ { key: 'e', label: say('SAY_EPOCHS'), min: 0, max: EMAX, step: STEP },
+], { names: W.SAY_PRESET_NAMES, values: { e: K('K_PRESET_EPOCHS', []) } }, draw)
+
+const svg = canvas('nn-curve', say('SAY_CURVE_LABEL'))
+const grid = canvas('nn-heat', say('SAY_GRID_LABEL'))
+const stat = el('div')
+root.replaceChildren(ask(say('SAY_ASK')), ctl.node, el('div', { class: 'nn-two' }, el('div', {}, el('p', { class: 'dk-label' }, say('SAY_GRID_LABEL')), grid), el('div', {}, el('p', { class: 'dk-label' }, say('SAY_CURVE_LABEL')), svg)), stat, el('p', { class: 'dk-note' }, say('SAY_NOTE')), source())
+
+// The loss after every STEP epochs, trained once and kept: each point is a full run of the spec.
+let curve = null
+const lossCurve = () => (curve ??= [...Array(EMAX / STEP)].map((_, k) => [(k + 1) * STEP, xor((k + 1) * STEP, 1)]))
+const lg = (v) => Math.log10(Math.max(v, 10 ** LOG0))
+
+function draw(s) {
+ const cases = K('K_CASES', [])
+ const out = [0, 1, 2, 3].map((c) => (s.e > 0 ? xor(s.e, 2 + c) : xor(0, 2 + c)))
+ heat(grid, { rows: 2, cols: 2, value: (r, c) => out[r * 2 + c], color: (v) => heatTone(v), rowLabels: [`${say('SAY_A')} 0`, `${say('SAY_A')} 1`], colLabels: [`${say('SAY_B')} 0`, `${say('SAY_B')} 1`], labelWidth: 60, w: 400, cell: 150, fmt: (v) => fmt(v, 3) })
+ svg.replaceChildren()
+ const o = { w: 640, h: 330, x0: 0, x1: EMAX, y0: LOG0, y1: LOG1, xticks: K('K_EPOCH_TICKS', []), yticks: K('K_LOG_TICKS', []), yfmt: (t) => `1e${t}`, xlabel: say('SAY_EPOCHS') }
+ const p = plot(svg, o)
+ p.line(lossCurve().map(([e, v]) => [e, lg(v)]), COLORS[6], 3)
+ const now = s.e > 0 ? xor(s.e, 1) : xor(1, 0)
+ cursor(svg, p, s.e, [[lg(now), COLORS[1]]])
+ dragX(svg, p, o, (e) => ctl.set('e', Math.round(e / STEP) * STEP))
+ svg.append(svgEl('text', { x: p.x(EMAX) - 6, y: p.top + 16, class: 'dk-txt-dim', 'text-anchor': 'end' }, say('SAY_LOG_NOTE')))
+ stat.replaceChildren(tiles([
+ { label: say('SAY_FIRST_LOSS'), value: fmt(xor(Math.max(1, s.e), 0), 4), tone: 6 },
+ { label: say('SAY_LAST_LOSS'), value: s.e > 0 ? xor(s.e, 1).toExponential(2) : say('SAY_UNTRAINED'), tone: 1 },
+ { label: say('SAY_OUTPUTS'), value: out.map((v) => fmt(v, 2)).join(' '), tone: 0 },
+ { label: say('SAY_TARGETS'), value: cases.join(' '), tone: 2 },
+ ]))
+}
+
+ctl.start()
diff --git a/apps/website/scripts/agents-from-specs.mjs b/apps/website/scripts/agents-from-specs.mjs
index 8fada5eeb1..c231a52608 100644
--- a/apps/website/scripts/agents-from-specs.mjs
+++ b/apps/website/scripts/agents-from-specs.mjs
@@ -340,10 +340,12 @@ export function literalValue(expr, name = 'literal') {
if (expr.value === undefined) throw new Error(`${name}: literal carries no value`)
if (raw === 'true') return true
if (raw === 'false') return false
- // Integers and decimal floats. Hex, binary expressions, struct literals and
- // calls appear elsewhere in the corpus and are deliberately still refused:
- // this function reads declarations, it does not evaluate them.
- if (/^-?\d+(\.\d+)?$/.test(raw)) return Number(raw)
+ // Integers and decimal floats, with an optional exponent (`1e-8`, `-1.0e30`: the
+ // epsilons of t27 specs/ml, which the course 10 widgets run). Hex, binary
+ // expressions, struct literals and calls appear elsewhere in the corpus and are
+ // deliberately still refused: this function reads declarations, it does not
+ // evaluate them.
+ if (/^-?\d+(\.\d+)?([eE][-+]?\d+)?$/.test(raw)) return Number(raw)
throw new Error(`${name}: unsupported literal ${JSON.stringify(raw)}`)
}
diff --git a/apps/website/scripts/agents-from-specs.test.mjs b/apps/website/scripts/agents-from-specs.test.mjs
index 2ba35c271b..3a4b6f3c5e 100644
--- a/apps/website/scripts/agents-from-specs.test.mjs
+++ b/apps/website/scripts/agents-from-specs.test.mjs
@@ -100,6 +100,15 @@ test('a negative number inside an array is read; a minus over anything else is r
assert.throws(() => constsOf(analyze('module m;\npub const N : [1]i32 = [-K];\npub const K : i32 = 1;\n')), /unsupported|minus/)
})
+test('a float with an exponent is read as its number; hex is still refused', () => {
+ // t27 specs/ml writes its epsilons this way (adam.t27 DEFAULT_EPSILON = 1e-8, attention NEG_INF = -1.0e30).
+ const c = constsOf(analyze('module m;\npub const E : f32 = 1e-8;\npub const F : f32 = 1.0e-5;\npub const G : f32 = -1.0e30;\n'))
+ assert.equal(c.E.value, 1e-8)
+ assert.equal(c.F.value, 1e-5)
+ assert.equal(c.G.value, -1e30)
+ assert.throws(() => constsOf(analyze('module m;\npub const H : u32 = 0xff;\n')), /unsupported/)
+})
+
test('signed integers have a shape: in range passes, out of range and the wrong type do not', () => {
const c = constsOf(analyze('module m;\npub const T : [3]i32 = [-1, 0, 1];\npub const N : i32 = -5;\npub const B : i8 = -129;\npub const U : u8 = 3;\n'))
assert.equal(shapeProblem(c.T, 'arr-i32'), null)
diff --git a/apps/website/scripts/course-from-spec.test.mjs b/apps/website/scripts/course-from-spec.test.mjs
index 4a5ed9682f..0f14b9628e 100644
--- a/apps/website/scripts/course-from-spec.test.mjs
+++ b/apps/website/scripts/course-from-spec.test.mjs
@@ -119,7 +119,7 @@ test('a lesson in two courses fails the build: its page can show one', async ()
})
test('a course left out of the catalog fails the build', async () => {
- await fails({ catalogText: replaced(inputs.catalogText, 'pub const COURSE_COUNT : u8 = 9;', 'pub const COURSE_COUNT : u8 = 10;') }, 'the_basics_come_first')
+ await fails({ catalogText: replaced(inputs.catalogText, 'pub const COURSE_COUNT : u8 = 10;', 'pub const COURSE_COUNT : u8 = 11;') }, 'the_basics_come_first')
})
test('a broken claim in a spec fails its own test block', async () => {
diff --git a/apps/website/scripts/widgets-from-spec.mjs b/apps/website/scripts/widgets-from-spec.mjs
index 6eebbed17d..7c51919b54 100644
--- a/apps/website/scripts/widgets-from-spec.mjs
+++ b/apps/website/scripts/widgets-from-spec.mjs
@@ -33,7 +33,7 @@ const REQUIRED = {
TAB_PAGE: 'str', POSTS_NOTHING: 'bool',
TOOL_BRAND: 'str', TOOL_GALLERY_LINK: 'str', TOOL_SOURCE_LINK: 'str', TOOL_DATA_LABEL: 'str', TOOL_PRIVACY: 'str', TOOL_PRIVACY_FILES: 'str', TOOL_NOSCRIPT: 'str',
CATEGORY_COUNT: 'u8', CATEGORY_IDS: 'arr', CATEGORY_NAMES: 'arr', CATEGORY_LINES: 'arr', KINDS: 'arr',
- WIDGET_COUNT: 'u8', WIDGET_IDS: 'arr', WIDGET_KINDS: 'arr', WIDGET_CATEGORIES: 'arr', WIDGET_PAGES: 'arr',
+ WIDGET_COUNT: 'u16', WIDGET_IDS: 'arr', WIDGET_KINDS: 'arr', WIDGET_CATEGORIES: 'arr', WIDGET_PAGES: 'arr',
WIDGET_TITLES: 'arr', WIDGET_LINES: 'arr', WIDGET_HOOKS: 'arr',
IDEA_COUNT: 'u8', IDEA_IDS: 'arr', IDEA_TITLES: 'arr', IDEA_LINES: 'arr', IDEA_SOURCES: 'arr',
}
diff --git a/apps/website/specs/course/courses.t27 b/apps/website/specs/course/courses.t27
index c3e45d88e1..5bcd8cc900 100644
--- a/apps/website/specs/course/courses.t27
+++ b/apps/website/specs/course/courses.t27
@@ -28,10 +28,10 @@ pub const SCHEMA_VERSION : u8 = 1;
; nothing (#/verify-hardware had no route).
pub const GENERATED : [2]str = ["src/lib/course.generated.ts", "src/lib/courseRoutes.generated.ts"];
-pub const COURSE_COUNT : u8 = 9;
+pub const COURSE_COUNT : u8 = 10;
-pub const COURSE_IDS : [9]str = ["t27-basics", "course", "ai-numbers", "goldenfloat", "verify-hardware", "clocks-and-cdc", "buses-and-peripherals", "ternary-computing", "arithmetic-and-dsp"];
-pub const COURSE_SPECS : [9]str = ["specs/course/t27-basics.t27", "specs/course/course.t27", "specs/course/ai-numbers.t27", "specs/course/goldenfloat.t27", "specs/course/verify-hardware.t27", "specs/course/clocks-and-cdc.t27", "specs/course/buses-and-peripherals.t27", "specs/course/ternary-computing.t27", "specs/course/arithmetic-and-dsp.t27"];
+pub const COURSE_IDS : [10]str = ["t27-basics", "course", "ai-numbers", "goldenfloat", "verify-hardware", "clocks-and-cdc", "buses-and-peripherals", "ternary-computing", "arithmetic-and-dsp", "neural-networks"];
+pub const COURSE_SPECS : [10]str = ["specs/course/t27-basics.t27", "specs/course/course.t27", "specs/course/ai-numbers.t27", "specs/course/goldenfloat.t27", "specs/course/verify-hardware.t27", "specs/course/clocks-and-cdc.t27", "specs/course/buses-and-peripherals.t27", "specs/course/ternary-computing.t27", "specs/course/arithmetic-and-dsp.t27", "specs/course/neural-networks.t27"];
pub const LESSONS_PER_COURSE : u8 = 27;
pub const MODULES_PER_COURSE : u8 = 9;
@@ -50,7 +50,7 @@ test every_course_is_three_cubed {
}
test the_basics_come_first {
- assert COURSE_COUNT == 9;
+ assert COURSE_COUNT == 10;
assert COURSE_IDS[0] == "t27-basics";
assert COURSE_SPECS[0] == "specs/course/t27-basics.t27";
assert COURSE_IDS[1] == "course";
@@ -65,7 +65,12 @@ test the_ternary_machine_follows_the_buses {
assert COURSE_SPECS[7] == "specs/course/ternary-computing.t27";
}
-test arithmetic_and_dsp_closes_the_catalog {
+test arithmetic_and_dsp_follows_the_ternary_machine {
assert COURSE_IDS[8] == "arithmetic-and-dsp";
assert COURSE_SPECS[8] == "specs/course/arithmetic-and-dsp.t27";
}
+
+test neural_networks_close_the_catalog {
+ assert COURSE_IDS[9] == "neural-networks";
+ assert COURSE_SPECS[9] == "specs/course/neural-networks.t27";
+}
diff --git a/apps/website/specs/course/neural-networks-ru.t27 b/apps/website/specs/course/neural-networks-ru.t27
new file mode 100644
index 0000000000..ce9746e94c
--- /dev/null
+++ b/apps/website/specs/course/neural-networks-ru.t27
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/course/neural-networks-ru.t27 -- translation contract, Russian, for the course on neural networks
+; Specs are English-only (LANG-EN, L3). This module carries no translated word: it
+; says WHICH fields of specs/course/neural-networks.t27 the Russian bundle translates, WHERE
+; the bundle lives, HOW it is keyed and what the page does when a field is missing.
+; The Cyrillic text is in the bundle named by BUNDLE_PATH, never in a .t27 file.
+; apps/website/scripts/course-from-spec.mjs refuses the build when the bundle misses
+; a field (COVERAGE_REQUIRED), carries a key the course does not have
+; (ORPHANS_ALLOWED), changes a number, or drops a {placeholder}.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module neural_networks_ru;
+
+pub const KIND : str = "i18n";
+pub const LOCALE : str = "ru";
+pub const SOURCE_LOCALE : str = "en";
+pub const SOURCE_SPEC : str = "specs/course/neural-networks.t27";
+pub const BUNDLE_REPO : str = "trinity";
+pub const BUNDLE_PATH : str = "apps/website/i18n/neural-networks.ru.json";
+pub const BUNDLE_FORMAT : str = "json";
+; A module entry is keyed by its MODULE_IDS value, a lesson entry by its LESSON_IDS
+; value; the page strings sit under the key UI_KEY, keyed by the SAY_ name.
+pub const MODULE_FIELDS : [2]str = ["TITLE", "LINE"];
+pub const LESSON_FIELDS : [4]str = ["TITLE", "GOAL", "TEXT", "TASK"];
+pub const UI_KEY : str = "ui";
+pub const FALLBACK : str = "en";
+; true: a module, lesson or page string with no Russian fails the build.
+pub const COVERAGE_REQUIRED : bool = true;
+; false: a bundle key the course does not have fails the build.
+pub const ORPHANS_ALLOWED : bool = false;
+pub const ENABLED : bool = true;
+
+test the_course_is_translated_whole {
+ assert COVERAGE_REQUIRED == true;
+ assert ORPHANS_ALLOWED == false;
+ assert LOCALE == "ru";
+ assert FALLBACK == "en";
+}
diff --git a/apps/website/specs/course/neural-networks.t27 b/apps/website/specs/course/neural-networks.t27
new file mode 100644
index 0000000000..1d004ddf19
--- /dev/null
+++ b/apps/website/specs/course/neural-networks.t27
@@ -0,0 +1,380 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/course/neural-networks.t27 -- the t27 course on neural networks, from one neuron to a transformer block
+; Source of truth for apps/website/scripts/course-from-spec.mjs (gHashTag/trinity), which reads
+; it the way it reads specs/course/course.t27: the constant schema, every test block below, every
+; widget in specs/widgets/gallery.t27, every lesson spec compiled clean, the Russian bundle named
+; by specs/course/neural-networks-ru.t27. It writes this course into src/lib/course.generated.ts
+; (the page at #/neural-networks) and a byte-identical copy of this file at
+; public/learn/neural-networks.t27. ASCII only (L3), English only (LANG-EN).
+
+; WHAT THE COURSE IS: course 10 of specs/course/courses.t27, 27 lessons in 9 modules of 3: one
+; neuron, new bends, a layer, pictures, the error, downhill, learning, memory and attention, and
+; the transformer and the reward. Every lesson opens one interactive widget and one spec. The
+; agents of this project think with neural networks; this course opens their parts as the
+; gHashTag/t27 specs/ml specs they are written in, each with numeric tests. Every widget runs the
+; fn bodies of one of those specs compiled to WebAssembly (public/widgets//logic.js, held to
+; its spec by scripts/widget-logic.mjs), vendored byte for byte from t27 master 239b4b059; the
+; max-pool spec from 2e53089d5, which added out_side (gHashTag/t27#10665) for the nn-maxpool widget.
+
+; WHAT THE COURSE DOES NOT DO: it says no number of its own. Every number in a lesson is one the
+; lesson spec computes and its widget shows. Lesson 10 is average pooling, not a convolution: the
+; sliding conv2d spec (gHashTag/t27#10612) was not merged when the course was made, and the
+; lesson says so. Lesson 20 shows the four inputs of XOR, not a decision region over the plane.
+
+; WHY 27: 3 cubed lessons, the size of a TRI-27 word. Changing the shape changes the tests.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module neural_networks;
+
+pub const KIND : str = "course";
+pub const ID : str = "neural-networks";
+pub const SCHEMA_VERSION : u8 = 1;
+; The derived files the generator rewrites (relative to apps/website).
+pub const GENERATED : [2]str = ["src/lib/course.generated.ts", "public/learn/neural-networks.t27"];
+pub const ROUTE : str = "neural-networks";
+; Share pages: learn/ for course 1, learn// for the next (specs/course/courses.t27).
+pub const SHARE_PATH : str = "learn/neural-networks/";
+; The paid training every course page links to (specs/course/arithmetic-and-dsp.t27).
+pub const COHORT_ROUTE : str = "fpga-training";
+pub const GALLERY : str = "specs/widgets/gallery.t27";
+pub const LOCALES : [2]str = ["en", "ru"];
+pub const RU_CONTRACT : str = "specs/course/neural-networks-ru.t27";
+; Lesson marks the reader sets are kept in this browser only, under this key, one key for
+; every course.
+pub const PROGRESS_KEY : str = "t27-course-done";
+pub const SENDS_NOTHING : bool = true;
+; A lesson's widget is the page's main stage, so it is drawn taller than a
+; gallery card (specs/widgets/gallery.t27 EMBED_* heights are for embeds in posts).
+pub const TOOL_FRAME_HEIGHT : u16 = 760;
+pub const PLAYER_FRAME_HEIGHT : u16 = 480;
+
+; --- Words on the page ---------------------------------------------------------------------
+pub const TITLE : str = "Neural networks in t27: from one neuron to a transformer block";
+pub const DESCRIPTION : str = "Course 10, 27 lessons, a live widget and a t27 spec each: activations, softmax and embeddings, pooling and batch norm, losses, SGD, RMSProp and Adam, backpropagation and XOR, recurrent cells, attention, a transformer block and PPO. Every number is computed by a compiled spec.";
+pub const SAY_KICKER : str = "Course";
+pub const SAY_LEAD : str = "The agents of this project think with neural networks, and here their parts are written as t27 specs with tests, specs the compiler checks. Every widget runs the fn bodies of a specs/ml spec compiled to WebAssembly: move a slider and the spec answers.";
+pub const SAY_SHAPE : str = "9 modules of 3 lessons, 27 cells. A filled cell is a lesson you marked done.";
+pub const SAY_START : str = "Start lesson 1";
+pub const SAY_CONTINUE : str = "Continue";
+pub const SAY_MODULE : str = "Module {0}";
+pub const SAY_LESSON : str = "Lesson {0} of {1}";
+pub const SAY_GOAL : str = "You will learn";
+pub const SAY_TRY : str = "Try it";
+pub const SAY_ALSO : str = "Also try";
+pub const SAY_ALSO_HINT : str = "Each one swaps the widget above.";
+pub const SAY_BACK_TO_MAIN : str = "Back to this lesson's widget";
+pub const SAY_SPEC : str = "Open the lesson's spec in the player";
+pub const SAY_OPEN_PAGE : str = "Open the widget on its own page";
+pub const SAY_WIDGET_LANG : str = "Widgets keep their own English words: each one lives in its own t27 spec.";
+pub const SAY_PREV : str = "Previous";
+pub const SAY_NEXT : str = "Next";
+pub const SAY_ALL : str = "All lessons";
+pub const SAY_MARK : str = "Mark as done";
+pub const SAY_MARKED : str = "Done";
+pub const SAY_PROGRESS : str = "{0} of {1} done";
+pub const SAY_PRIVATE : str = "Progress stays in this browser; nothing is sent.";
+pub const SAY_SOURCE : str = "This course is itself a t27 spec: read it";
+pub const SAY_COHORT : str = "Looking for the paid FPGA training? It moved to its own page.";
+pub const SAY_NOT_FOUND : str = "No lesson has this address.";
+pub const SAY_OPEN_LESSON : str = "Open the interactive lesson";
+pub const SAY_OPEN_COURSE : str = "Open the interactive course";
+pub const SAY_SEO_TITLE : str = "{0}: neural networks course, lesson {1} of {2}";
+pub const SAY_SHARE : str = "Link to share, with a preview card";
+pub const SAY_NEXT_COURSE : str = "Next course";
+pub const SAY_PREV_COURSE : str = "Previous course";
+
+; --- Modules -------------------------------------------------------------------------------
+pub const LESSONS_PER_MODULE : u8 = 3;
+pub const MODULE_COUNT : u8 = 9;
+pub const MODULE_IDS : [9]str = ["one-neuron", "new-bends", "a-layer", "pictures", "the-error", "downhill", "learning", "memory-and-attention", "transformer-and-reward"];
+pub const MODULE_TITLES : [9]str = [
+ "One neuron",
+ "New bends",
+ "A layer",
+ "Pictures",
+ "The error",
+ "Downhill",
+ "Learning",
+ "Memory and attention",
+ "The transformer and the reward"
+];
+pub const MODULE_LINES : [9]str = [
+ "A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back.",
+ "The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff.",
+ "What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off.",
+ "Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales.",
+ "How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers.",
+ "Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them.",
+ "Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches.",
+ "Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once.",
+ "The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching."
+];
+
+; --- Lessons -------------------------------------------------------------------------------
+; LESSON_WIDGETS names the widget the lesson opens; every one is a gallery id, and no two
+; lessons of any course open the same one. Each module's three widgets are offered in each
+; other's LESSON_ALSO, next to related widgets of this course and of the ternary course.
+; LESSON_SPECS is the spec under public/t27/files/ the lesson opens in the player, and the spec
+; whose compiled fn bodies the widget runs.
+pub const LESSON_COUNT : u8 = 27;
+pub const LESSON_IDS : [27]str = [
+ "relu", "the-sigmoid", "tanh-and-saturation",
+ "gelu", "swish-and-silu", "elu",
+ "softmax-and-temperature", "embeddings", "dropout",
+ "patches-of-a-picture", "max-pooling", "batch-normalisation",
+ "mean-squared-error", "cross-entropy", "the-huber-loss",
+ "gradient-descent", "rmsprop", "adam",
+ "backpropagation", "learning-xor", "lamb-and-large-batches",
+ "a-recurrent-cell", "lstm-gates", "attention",
+ "positional-encoding", "an-encoder-block", "ppo-and-the-reward"
+];
+pub const LESSON_MODULES : [27]str = [
+ "one-neuron", "one-neuron", "one-neuron",
+ "new-bends", "new-bends", "new-bends",
+ "a-layer", "a-layer", "a-layer",
+ "pictures", "pictures", "pictures",
+ "the-error", "the-error", "the-error",
+ "downhill", "downhill", "downhill",
+ "learning", "learning", "learning",
+ "memory-and-attention", "memory-and-attention", "memory-and-attention",
+ "transformer-and-reward", "transformer-and-reward", "transformer-and-reward"
+];
+pub const LESSON_WIDGETS : [27]str = [
+ "nn-relu", "nn-sigmoid", "nn-tanh",
+ "nn-gelu", "nn-silu", "nn-elu",
+ "nn-softmax", "nn-embedding", "nn-dropout",
+ "nn-conv", "nn-maxpool", "nn-batchnorm",
+ "nn-mse", "nn-bce", "nn-huber",
+ "nn-sgd", "nn-rmsprop", "nn-adam",
+ "nn-backprop", "nn-xor", "nn-lamb",
+ "nn-rnn", "nn-lstm", "nn-attention",
+ "nn-positions", "nn-encoder", "nn-ppo"
+];
+pub const LESSON_ALSO : [27]str = [
+ "nn-sigmoid,nn-tanh,tc-one-neuron",
+ "nn-relu,nn-tanh,nn-bce",
+ "nn-relu,nn-sigmoid",
+ "nn-silu,nn-elu,nn-encoder",
+ "nn-gelu,nn-elu,nn-relu",
+ "nn-relu,nn-silu,nn-gelu",
+ "nn-embedding,nn-dropout,nn-bce",
+ "nn-softmax,nn-dropout,nn-positions",
+ "nn-softmax,nn-embedding",
+ "nn-maxpool,nn-batchnorm,tc-detector",
+ "nn-conv,nn-batchnorm",
+ "nn-conv,nn-maxpool,nn-encoder",
+ "nn-bce,nn-huber",
+ "nn-mse,nn-huber,nn-sigmoid",
+ "nn-mse,nn-bce",
+ "nn-rmsprop,nn-adam,nn-mse",
+ "nn-sgd,nn-adam",
+ "nn-sgd,nn-rmsprop,nn-lamb",
+ "nn-xor,nn-lamb,tc-tiny-layer",
+ "nn-backprop,nn-lamb,tc-tiny-layer",
+ "nn-adam,nn-xor,nn-backprop",
+ "nn-lstm,nn-attention",
+ "nn-rnn,nn-attention,nn-sigmoid",
+ "nn-rnn,nn-lstm,nn-encoder",
+ "nn-attention,nn-encoder,nn-embedding",
+ "nn-attention,nn-positions,nn-batchnorm",
+ "nn-softmax,nn-adam,nn-encoder"
+];
+pub const LESSON_SPECS : [27]str = [
+ "specs/ml/activation/relu_activation.t27",
+ "specs/ml/activation/sigmoid_activation.t27",
+ "specs/ml/activation/tanh_activation.t27",
+ "specs/ml/activation/gelu_activation.t27",
+ "specs/ml/activation/silu_swish_activation.t27",
+ "specs/ml/activation/elu_activation.t27",
+ "specs/ml/activation/softmax.t27",
+ "specs/ml/layers/embedding_layer.t27",
+ "specs/ml/layers/dropout_layer.t27",
+ "specs/ml/layers/avgpool2d_layer.t27",
+ "specs/ml/layers/maxpool2d_layer.t27",
+ "specs/ml/layers/batchnorm_layer.t27",
+ "specs/ml/loss/mse_loss.t27",
+ "specs/ml/loss/binary_crossentropy_loss.t27",
+ "specs/ml/loss/huber_loss.t27",
+ "specs/ml/optimizer/sgd.t27",
+ "specs/ml/optimizer/rmsprop.t27",
+ "specs/ml/optimizer/adam.t27",
+ "specs/ml/layers/dense_layer.t27",
+ "specs/ml/pathway/mlp.t27",
+ "specs/ml/optimizer/lamb.t27",
+ "specs/ml/recurrent/rnn_cell.t27",
+ "specs/ml/recurrent/lstm_single.t27",
+ "specs/ml/recurrent/attention_mechanism.t27",
+ "specs/ml/transformer/positional_encoding.t27",
+ "specs/ml/transformer/encoder_block.t27",
+ "specs/ml/rl/ppo_clip_loss.t27"
+];
+pub const LESSON_TITLES : [27]str = [
+ "ReLU: keep the positive part",
+ "The sigmoid: a number into a probability",
+ "tanh and saturation",
+ "GELU: a dip below zero",
+ "Swish and SiLU",
+ "ELU: a floor, not a cliff",
+ "Softmax and temperature",
+ "Embeddings",
+ "Dropout",
+ "Patches of a picture",
+ "Max pooling",
+ "Batch normalisation",
+ "Mean squared error",
+ "Cross-entropy",
+ "The Huber loss",
+ "Gradient descent",
+ "RMSProp",
+ "Adam",
+ "Backpropagation",
+ "Learning XOR",
+ "LAMB and large batches",
+ "A recurrent cell",
+ "LSTM gates",
+ "Attention",
+ "Positional encoding",
+ "An encoder block",
+ "PPO and the reward"
+];
+pub const LESSON_GOALS : [27]str = [
+ "What ReLU does to a positive and to a negative input, and why a slope of 0 stops a unit from learning.",
+ "How the sigmoid maps any input to a probability, and why its slope limits deep networks.",
+ "Why tanh is centred on zero, and where its slope falls so low that a unit stops learning.",
+ "What GELU computes, why it dips below zero, and how close its tanh shortcut comes.",
+ "How x times sigmoid(beta x) gates itself, and how one parameter moves it from a smooth curve to ReLU.",
+ "How ELU keeps a soft negative side with a floor at -alpha, and why its slope never quite dies.",
+ "How softmax turns scores into probabilities that sum to one, and what the temperature does to them.",
+ "How a token becomes a row of numbers, and why learning changes only the rows that were read.",
+ "Why switching units off while training helps, and why the survivors must be scaled.",
+ "Why networks read a picture in patches, and what a 2 x 2 window averaged over each patch gives.",
+ "What max pooling keeps, where it remembers taking it from, and where the gradient goes.",
+ "How batch norm puts features on different scales on one scale, and what it uses at inference.",
+ "How MSE turns every miss into one number, why it chases large errors, and where its gradient points.",
+ "Why cross-entropy charges a confident wrong answer so much, and why that is what trains a classifier.",
+ "How the Huber loss treats a small error like squared error and a large one linearly, so outliers pull less.",
+ "How gradient descent walks down a loss, what momentum adds, and why too large a rate never settles.",
+ "How RMSProp divides each step by a running root mean square, so the step no longer depends on the gradient's size.",
+ "How Adam combines momentum with a running scale, and why its bias correction makes step 1 a fixed size.",
+ "How the gradient of each weight and each input of a layer follows from the gradient of its outputs.",
+ "Why one neuron cannot learn XOR, and how a hidden layer trained by backpropagation does.",
+ "How LAMB scales each layer's step by its own weight norm, so one learning rate suits layers of every size.",
+ "How a recurrent cell carries a state from step to step, and why that memory fades.",
+ "How the forget, input and output gates of an LSTM decide what the cell keeps, writes and shows.",
+ "How attention scores every query against every key, what a softmax over each row gives, and what the causal mask removes.",
+ "How sines and cosines at falling frequencies give every position its own vector, and why a shift is a turn.",
+ "What one transformer encoder block does to its input, and what moving the norms in front changes.",
+ "How PPO turns a reward into a policy update, and how its clip keeps one update from going too far."
+];
+pub const LESSON_TEXTS : [27]str = [
+ "A neuron adds up its weighted inputs and passes the sum through an activation. ReLU is the plainest one, max(0, x): a positive sum goes through unchanged and a negative one becomes 0. Its slope is 1 or 0, and a slope of 0 is a unit that has stopped learning, because no gradient flows back through it. A leaky slope fixes that: with a slope of 0.1, x = -2 gives -0.2 and passes back a gradient of 0.1. The spec relu_activation.t27 computes both curves the widget draws.",
+ "The sigmoid 1 / (1 + e^-x) squeezes the whole number line into 0 to 1, which is why it reads as a probability: 2 becomes 0.8808 and -4 becomes 0.018. Its slope is s (1 - s), computed from the output s itself, and it is largest at 0, where it is 0.25; at x = 8 it is 0.000335. A gradient that passes back through a stack of sigmoids is multiplied by at most 0.25 at each one, which is why deep sigmoid networks trained slowly. The spec sigmoid_activation.t27 computes the curve and the slope.",
+ "tanh is a sigmoid stretched to -1..1 and centred on 0, so a layer of tanh units passes on values that average near zero, and its slope reaches 1 at the centre. It still saturates: at 2.5 the output is 0.9866 and the slope only 0.0266. The red band marks every input where the slope is under 0.1, beyond about 1.8 on either side, and a unit whose inputs live there barely learns. The spec tanh_activation.t27 computes the curve, the slope and its own batch, tanh of 0, 1 and 2: 0, 0.762 and 0.964.",
+ "GELU weighs x by the chance that a standard normal draw is below it: GELU(x) = x Phi(x). A large positive input passes, a large negative one vanishes, and in between the curve dips to about -0.17 near x = -0.75 before it rises. BERT and GPT use it in their feed-forward layers. The exact form needs erf, which the spec computes with the Abramowitz-Stegun fit, good to 1.5e-7, and it also computes the tanh shortcut: at -0.75 they give -0.16997 and -0.17004. The spec gelu_activation.t27 computes both curves and their slopes.",
+ "Swish is x times sigmoid(beta x), a unit that gates itself. At beta 1 it is called SiLU, and its lowest point is -0.2785 near x = -1.28. Raise beta and the gate sharpens: at beta 8 the curve is ReLU with rounded corners; at beta 0.5 it is a gentle slope through zero. One parameter moves the unit between a smooth curve and a switch, and a network can learn it. The spec silu_swish_activation.t27 computes the curve and its slope for any beta, and its own batch at -2, 0 and 2: -0.238, 0 and 1.762.",
+ "ELU keeps the positive side of ReLU and replaces the cliff on the negative side with alpha (e^x - 1), a curve that flattens toward -alpha. With alpha 1, x = -3 gives -0.9502, already near the floor of -1, and the slope there is 0.0498: small, but not 0, so the unit can still move. Because the negative outputs are not thrown away, a layer's outputs average closer to 0. The spec elu_activation.t27 computes the curve, the slope and the floor, and its own batch at -1, 0 and 2: -0.632, 0 and 2.",
+ "A classifier ends with one score per class, and softmax turns the scores into probabilities: exp of each score over the sum of all of them. Scores 1, 2 and 3 become 0.090, 0.245 and 0.665. Dividing the scores by a temperature first changes how sure it is: at 0.5 the top score takes 0.867, at 5 the three are 0.27, 0.33 and 0.40. A language model picks its next token from exactly this, and the temperature setting of a chat model is this T. The spec softmax.t27 subtracts the largest score first, so exp never overflows.",
+ "A network cannot multiply a word, so each token is an index into a table with one row of numbers per token, and looking a token up copies its row. In the spec the table has 4 tokens of width 2 and row v holds (10 v, 10 v + 1), so token 2 reads (20, 21). Learning sends the gradient back only to the rows that were read, and a token read twice collects both: gradient rows (1, 2) and (3, 4) on token 2 add up to (4, 6). The spec embedding_layer.t27 computes the lookup and the gradient.",
+ "While training, dropout switches off a share p of the units at random, so the rest cannot lean on any one of them. The units that survive must then be scaled, or the next layer would see the signal at another size in training than at inference. Inverted dropout scales them by 1 / (1 - p) while training: with p = 0.5, input 1, 2, 3, 4 becomes 2, 0, 6, 0, and inference is a plain pass. Classic dropout keeps 1, 0, 3, 0 and scales by 0.5 at inference instead. The spec dropout_layer.t27 uses a fixed mask, keep, drop, keep, drop, so the numbers are the same every time.",
+ "A picture is a grid of numbers, and networks read it in small patches so that the same detector works anywhere in it. Here a 2 x 2 window steps by 2 over a 4 x 4 image whose cell i holds i, and each output is the average of its patch: 2.5, 4.5, 10.5 and 12.5. In the backward pass each output's gradient is split equally over its 4 cells. This is average pooling: a convolution would weight the 4 cells with a learned kernel, and its spec, t27#10612, is not merged yet. The spec avgpool2d_layer.t27 computes both passes.",
+ "Max pooling keeps the largest value of each window and drops the rest, so the network cares less about exactly where a feature sits. On the spec's 4 x 4 grid, 2 x 2 windows with stride 2 keep 4, 8, 9 and -2, taken from cells 5, 6, 13 and 11. Overlapping 3 x 3 windows with stride 1 keep 8, 8, 9 and 9. In the backward pass each window's whole gradient goes to its maximum: gradients 1, 2, 3 and 4 land on cells 5, 6, 13 and 11, and every other cell gets 0. The spec maxpool2d_layer.t27 computes the outputs, their positions and the gradient.",
+ "Features on different scales pull a network around: in the spec's batch one feature is ten times another. Batch normalisation subtracts each feature's batch mean and divides by its spread, so the features 1, 2, 3, 4 and 10, 20, 30, 40 both come out as -1.34, -0.45, 0.45 and 1.34, and -1, -1, 1, 1 comes out as -1 and 1. A learned gamma and beta then scale and shift it back where that helps. At inference the layer uses running averages instead of the batch: with mean 0, variance 1, gamma 1 and beta 0.25, the inputs 1, 2 and 3 give 1.25, 2.25 and 3.25. The spec batchnorm_layer.t27 computes both.",
+ "A loss turns all the misses of a model into one number to push down. Mean squared error averages the squared misses: predictions 1.5, 2 and 2 against targets 1, 2 and 3 cost (0.25 + 0 + 1) / 3 = 0.4167. Squaring makes a miss of 2 cost 4 times a miss of 1, so MSE chases large errors. Its gradient for each prediction is 2 (p - t) / n, here 0.333, 0 and -0.667: each prediction is pushed back toward its target. As a function of one prediction the loss is a parabola, a bowl with one bottom. The spec mse_loss.t27 computes the loss and the gradient.",
+ "A classifier outputs a probability, and cross-entropy charges -log of the probability it gave to the true answer. Predictions 0.8, 0.3 and 0.9 against answers 1, 0 and 1 cost 0.2284 on average. Being sure and right is nearly free: 0.99, 0.01 and 0.99 cost 0.0101. Being sure and wrong is expensive: 0.01, 0.99 and 0.01 cost 4.6052, and toward 0 the cost grows without bound, which is why the spec clips p to [1e-7, 1 - 1e-7]. Its gradient is largest exactly where the model is sure and wrong. The spec binary_crossentropy_loss.t27 computes the loss and the gradient, from probabilities or from logits.",
+ "Squared error lets one outlier dominate: an error of 3 costs 4.5 as half its square, as much as 36 errors of 0.5. Huber is half the square while |e| is within delta and then grows only linearly, delta |e| - delta^2 / 2, so with delta 1 that error costs 2.5. The two pieces meet with the same value and slope, and the gradient never grows past delta: a far-off point pulls on the model with a bounded force. For the spec's pair of errors 0.5 and 3 the mean is 1.3125, against 2.3125 for half the square. The spec huber_loss.t27 computes both.",
+ "Training is walking downhill on the loss. Gradient descent moves a weight by -lr times its gradient each step; in the spec the loss is (x - 3)^2 from x = 0, and lr 0.1 reaches 2.99996 in 50 steps. Momentum adds a share of the last move, which speeds up a long slope and overshoots a sharp bottom: at 0.9 the weight shoots past 3 to about 5.13 and rings down to 3.09 after 50 steps. Nesterov takes the gradient one step ahead and swings less, to about 4.25. Too large a rate never settles: at lr 1 the weight bounces between 0 and 6 for good. The spec sgd.t27 computes every step.",
+ "The right step size depends on the size of the gradient, and different weights see gradients a million times apart. RMSProp keeps a running mean of the squared gradient, the cache, and divides each step by its root, so the step has no units: with a constant gradient of 0.001, 0.5 or 1000 the weight moves by exactly the same 0.0316 on step 1 and walks the same path, reaching -0.0914 after 100 steps at lr 0.01. The decay sets how slowly the cache fills: at 0.99 the same 100 steps end at -1.0261. The spec rmsprop.t27 computes the weight and the cache.",
+ "Adam combines the two ideas: m, a running mean of the gradient, as in momentum, and v, a running mean of its square, as in RMSProp; the step is lr m / sqrt(v). Both start at 0 and are too small early on, so Adam divides them by 1 - beta^t, and on step 1 that makes the move exactly lr times the sign of the gradient: gradients 2, -40 and 0.5 all move their weights by 0.001. On the spec's bowl (x - 3)^2 at lr 0.1, Adam rises from 0, overshoots to about 3.18 and settles at 3.00005 by step 200. The spec adam.t27 computes every step.",
+ "To learn, every weight needs to know how the loss changes when it moves. Backpropagation answers one layer at a time: given g, how much the loss wants each output to move, the weight from input i to output o gets g_o x_i, and input i gets the sum over o of W[o][i] g_o, which it hands on to the layer below. With inputs (1, 2, 3) and g = (1, 0.5) the weight gradients are 1, 2, 3 and 0.5, 1, 1.5, and the input gradients 0.5, 2 and 3.5. The layer is linear, so moving x0 by 1 moves the loss g . y from 15.25 to 15.75, by its gradient 0.5. The spec dense_layer.t27 computes all three.",
+ "XOR is 1 when exactly one of two inputs is 1. No straight line separates the cases that give 1 from those that give 0, so a single neuron cannot learn it. A hidden layer bends the space: the spec trains a 2-8-1 network of ReLUs by backpropagation, lr 0.05, from fixed initial weights. Its loss falls from 0.8896 in epoch 1 to 1.8e-13 after 2000 epochs, and its four answers come out within 4e-7 of 0, 1, 1 and 0. The widget runs that training in your browser, from scratch for every point of the curve. The spec mlp.t27 holds the network and the test that it learns.",
+ "Training with very large batches needs a large learning rate, and one rate can be too large for a small layer and too small for a big one. LAMB takes an Adam-style update for each layer and rescales it by the trust ratio, the layer's weight norm, capped at 10, over the update's norm, so every layer moves by the same fraction of itself. In the spec, the weights (1, 2) times a scale take a step whose norm is 0.001 times theirs: 0.00447 at scale 2, and once the weight norm passes 10, near scale 4.47, the step stays at 0.01. The spec lamb.t27 computes the update for one layer and for two at once.",
+ "A sentence comes one word at a time, and a recurrent cell carries a state h from one step to the next: h' = tanh(W_x x + W_h h + b). In the spec's tiny cell W_x is 1 and W_h 0.5, so a single pulse gives 0.7616, 0.3634 and 0.1797: the memory of it roughly halves each step. A second pulse at step 3 lifts h to 0.7968, a little above where the single pulse began, because the old memory is still there. After 8 steps only 0.0466 is left. That fading is why plain cells forget long contexts. The spec rnn_cell.t27 computes each step.",
+ "An LSTM cell keeps a separate memory c and three gates, each a sigmoid between 0 and 1: forget scales the old memory, input lets in new content g, and output decides how much of the memory to show. c' = f c + i g and h' = o tanh(c'). In the spec's fixture every weight is 0, so each gate is the sigmoid of its bias: with the forget gate at 0.9975 and the input gate at 0.0025, a memory of 1 comes out as 0.9975, and the output is 0.7587. Because c passes through with only a multiply by f, its gradient survives many steps too. The spec lstm_single.t27 computes the cell.",
+ "Attention lets each position decide which other positions to read. Every query is scored against every key, q . k / sqrt(d), and a softmax turns each row of scores into weights that sum to 1; the output is the values averaged by those weights. Two heads do it on the two halves of the vector. With a causal mask a position cannot read the future: in head 0, position 0 reads only itself, position 1 splits 0.452 and 0.548, and position 2 spreads 0.230, 0.321 and 0.449. Without the mask every row spreads over all three keys, from 0.21 to 0.47. The spec attention_mechanism.t27 computes every weight on its own fixture.",
+ "Attention treats its input as a set, so the order of the words has to be added to them. The sinusoidal encoding gives each position d numbers: the sin and cos of the position times a frequency that falls by 10000^(2/d) from pair to pair. With d = 8 the four pairs turn by 1, 0.1, 0.01 and 0.001 radians per position, like the hands of four clocks: position 5 reads -0.959, 0.284 in its fast pair and 0.005, 1.000 in its slow one. Each pair has length 1, and a shift by k positions turns every pair by its own fixed angle. The spec positional_encoding.t27 computes every value.",
+ "A transformer layer is two sublayers, each wrapped in a residual add and a layer norm: self-attention mixes the positions, then a feed-forward net of width 8 works on each position alone. In the post-norm order of the original paper the block ends with a norm, so every output row has mean 0 and variance 1.000. Pre-norm puts the norms in front of each sublayer and ends on a residual add, so its rows come out at variances 2.73, 3.33 and 0.96; deep stacks train more easily that way, with one norm added at the very top. The spec encoder_block.t27 runs one block on 3 positions, d_model 4 and 2 heads.",
+ "An agent learns from reward by making good actions more likely. PPO measures a change of policy by the ratio r of the new probability of an action to the old one and multiplies it by the advantage A, how much better the action did than expected. Then it clips: once r has moved more than eps the way A rewards, the objective goes flat. At r = 1.5, A = 1 and eps 0.2 the raw r A is 1.5, but PPO counts 1.2, so one batch cannot push the policy far. On the spec's batch, ratios 1, 1.1 and 0.9 with advantages 2, 1.5 and 3, the policy loss is -2.1167. The spec ppo_clip_loss.t27 computes it."
+];
+pub const LESSON_TASKS : [27]str = [
+ "Put x at -2 and read the slope with and without the leak; then find the slope at x = 1.5.",
+ "Find the input where the probability is 0.5; then drag to 6 and read how little slope is left.",
+ "Drag x until the band starts and read the slope there; then compare the slope at 0 and at 2.5.",
+ "Find the lowest point of the curve by its slope; then compare the exact and the tanh form at -3.",
+ "Hold x at -1.28 and raise beta from 1 to 8; then find the beta at which the dip is gone at this scale.",
+ "Set alpha to 0.5 and read the floor; then find how far left x must go before the slope is under 0.01.",
+ "Cool the temperature to 0.1 and read the three probabilities; then make a tie and heat it.",
+ "Read token 2 twice and then tokens 1 and 3; count the rows whose gradient is not 0 each time.",
+ "Compare the outputs of the four modes; then read the gradient of a dropped unit in training.",
+ "Tap each patch and check its average by eye; then switch to the gradient and read what each cell of window 4 receives.",
+ "Switch to 3 x 3 windows and find the cell two outputs share; then turn on the gradient for 2 x 2 windows.",
+ "Compare the two normalised columns; then set the running mean 2 and variance 4 and read the inference outputs.",
+ "Move prediction 2 onto its target and read the loss; then drag along the bowl and find its bottom.",
+ "Drag prediction 0 toward 0.01 and watch the curve; then read the gradient bars at 0.5, 0.5 and 0.5.",
+ "Set delta to 0.5 and drag the error to 3; then find where the two curves part.",
+ "Find the largest learning rate that still settles without momentum; then add momentum 0.9 and count the swings.",
+ "Switch between the three gradients and check that the lines do not move; then raise the decay to 0.99.",
+ "Give the three gradients very different sizes and read the weights after step 1; then find where the path tops out.",
+ "Set g = (1, 0) and read which edges go dark; then set g = (-2, 1) and check the input gradients by hand.",
+ "Drag the epochs back to 100 and read the four answers; then find the epoch where the curve stops falling.",
+ "Read the step at scales 1 and 2 and compare; then drag past the bend and explain why the line goes flat.",
+ "Load one pulse and read how many steps it takes to fall under 0.1; then make the input alternate.",
+ "Load Remember and then Forget and compare the new memory; then write with the input gate open.",
+ "Turn the mask off and on and read row 0; then compare the two heads on row 2.",
+ "Step from position 0 to 1 and watch which clock moves; then jump to 37 and find the pair that still points up.",
+ "Switch between post-norm and pre-norm and read the row variances; then find the position head 1 reads most.",
+ "Make the advantage negative and find where the curve goes flat now; then widen eps to 0.5."
+];
+
+; --- Claims --------------------------------------------------------------------------------
+test the_course_is_three_cubed {
+ assert LESSONS_PER_MODULE == 3;
+ assert MODULE_COUNT == 9;
+ assert LESSON_COUNT == 27;
+ assert MODULE_COUNT * LESSONS_PER_MODULE == LESSON_COUNT;
+ assert LESSON_COUNT == 3 * 3 * 3;
+}
+
+test it_starts_at_one_neuron_and_ends_with_a_reward {
+ assert MODULE_IDS[0] == "one-neuron";
+ assert LESSON_WIDGETS[0] == "nn-relu";
+ assert LESSON_SPECS[0] == "specs/ml/activation/relu_activation.t27";
+ assert MODULE_IDS[8] == "transformer-and-reward";
+ assert LESSON_WIDGETS[26] == "nn-ppo";
+ assert LESSON_SPECS[26] == "specs/ml/rl/ppo_clip_loss.t27";
+}
+
+test activations_come_before_layers {
+ assert MODULE_IDS[1] == "new-bends";
+ assert LESSON_WIDGETS[3] == "nn-gelu";
+ assert MODULE_IDS[2] == "a-layer";
+ assert LESSON_SPECS[6] == "specs/ml/activation/softmax.t27";
+}
+
+test the_picture_lesson_is_pooling_until_conv2d_slides {
+ assert MODULE_IDS[3] == "pictures";
+ assert LESSON_WIDGETS[9] == "nn-conv";
+ assert LESSON_SPECS[9] == "specs/ml/layers/avgpool2d_layer.t27";
+ assert LESSON_WIDGETS[10] == "nn-maxpool";
+}
+
+test losses_then_optimizers_then_learning {
+ assert MODULE_IDS[4] == "the-error";
+ assert MODULE_IDS[5] == "downhill";
+ assert LESSON_WIDGETS[17] == "nn-adam";
+ assert MODULE_IDS[6] == "learning";
+ assert LESSON_WIDGETS[19] == "nn-xor";
+ assert LESSON_SPECS[19] == "specs/ml/pathway/mlp.t27";
+}
+
+test memory_then_attention_then_the_transformer {
+ assert MODULE_IDS[7] == "memory-and-attention";
+ assert LESSON_WIDGETS[23] == "nn-attention";
+ assert LESSON_WIDGETS[25] == "nn-encoder";
+ assert LESSON_SPECS[25] == "specs/ml/transformer/encoder_block.t27";
+}
+
+test the_course_sends_nothing {
+ assert SENDS_NOTHING == true;
+ assert LOCALES[0] == "en";
+ assert LOCALES[1] == "ru";
+}
diff --git a/apps/website/specs/widgets/gallery.t27 b/apps/website/specs/widgets/gallery.t27
index 6285d9a4d1..17117ac2a9 100644
--- a/apps/website/specs/widgets/gallery.t27
+++ b/apps/website/specs/widgets/gallery.t27
@@ -78,15 +78,15 @@ pub const TOOL_NOSCRIPT : str = "This widget runs in your browser and needs Java
; Parallel arrays, WIDGET_COUNT long. WIDGET_TITLES is empty wherever the page carries an
; og:title, which is every kind but tab: the page is the one place the words live. A tool's
; WIDGET_HOOKS entry is empty too: its hook is the page's t27:hook, written from its own spec.
-pub const WIDGET_COUNT : u8 = 252;
-pub const WIDGET_IDS : [252]str = ["x7-board", "tri-x7-fasm", "tri-x7-mutate", "perf2-loop", "tri-fpga-rgmii", "tri-fpga-jtag", "tri-fpga-pins", "devkit-flow", "play-verilog", "bit-diff", "build-receipt", "cost-curves", "gf16-calc", "idcode", "lut-explain", "pin-map", "placement-map", "resource-bars", "slack-waterfall", "test-waves", "synth-smells", "bit-autopsy", "fasm-skyline", "vcd-wrapped", "fits-on", "stat-card", "lut-depth", "fsm-sketch", "lut-treemap", "t27-cell", "t27-vs-nvfp4", "uart-bucket", "tri-fpga-specs", "tri-fpga-tools", "tri-fpga-wire", "tri-fpga-batch", "tri-fpga-status", "tri-fpga-tmpcheck", "tri-fpga-keycheck", "tri-fpga-selftest", "tri-x7-refs", "tri-x7-audit", "t27c-clock-domain", "t27c-clocks", "t27c-wukong-v1", "t27c-cts", "t27c-mmcm", "t27c-mmcm-mutate", "t27c-timing", "t27c-reset-sync", "t27c-reset-sync-gen", "t27c-mtbf", "t27c-clock-domain-gen", "t27c-fifo", "t27c-fifo-mutate", "t27c-fifo-flags", "t27c-emitter-xdc", "t27c-fpga-rxcap", "t27-tri-competitors", "t27-tri-discard", "tri-fpga-txhold", "tri-fpga-rxcap", "tri-fpga-rxer", "tri-fpga-steps", "tri-fpga-ioclients", "tri-fpga-claim", "tri-fpga-release", "tri-fpga-next", "t27c-memory", "t27c-spi-tb", "t27c-check-uart", "t27c-check-spi", "t27c-check-apb", "t27c-check-axi4", "t27c-check-bridge", "t27c-check-memory", "t27c-verilog-uart", "t27c-verilog-spi", "t27c-verilog-apb", "t27c-verilog-axi4", "t27c-verilog-bridge", "t27c-verilog-top", "t27c-verilog-uart-tb", "t27c-verilog-apb-tb", "t27c-verilog-axi-tb", "t27c-verilog-mem-tb", "t27c-verilog-integration-tb", "t27c-verilog-top-tb", "t27c-verilog-spi-tb", "t27c-hir-uart", "t27c-hir-apb", "t27c-hir-axi4", "t27c-eth-crc", "t27c-rgmii", "jtag-verdict", "line-bounce", "eye-isi", "ecc-secded", "carry-chain", "prefix-adder", "csa-tree", "shift-add", "booth-radix4", "dsp-fit", "q-format", "round-sat", "sqnr", "cordic", "cordic-vector", "sine-table", "alias-fold", "nco", "dft-bin", "fir-response", "fir-design", "fir-cost", "decimator", "cic-growth", "polyphase", "fft-butterfly", "bit-reverse", "fft-scaling", "barker-corr", "fir-fit", "fir-on-board", "play-hello-world", "play-rust", "play-js", "t27c-hello-world", "t27-tri-misread", "test-touch", "t27c-stages", "t27c-e8m0", "play-e8m0", "t27c-ocp-mx", "play-gfternary", "t27c-gfternary", "play-golden-sieve", "t27c-golden-sieve", "play-tnf17", "t27c-tnf17", "t27c-activation-quantizer", "t27c-bitnet-majority", "t27c-bitnet-neuron-nchunk", "t27c-comb-ternary-dot", "t27c-stream-ternary-mac", "t27c-bitnet-mlp", "t27c-gft-smul", "t27c-gft-sadd", "t27c-gft-signed-mac", "t27c-gft-relu", "t27c-gft-exp2", "t27c-gft-argmax4", "t27c-gft-nll", "t27c-gft-sgd-step", "t27c-gft-xornet", "basics-what-a-spec-is", "basics-the-module-line", "basics-comments-and-prose", "basics-constants", "basics-whole-numbers", "basics-true-false-and-text", "basics-arrays", "basics-trits", "basics-expressions", "basics-a-test-block", "basics-many-tests", "basics-functions", "basics-local-names", "basics-if-and-else", "basics-loops", "basics-pub-or-private", "basics-structs", "basics-enums", "basics-use-other-modules", "basics-invariants", "basics-gen-ts", "basics-seven-backends", "basics-stages-of-t27c", "basics-tri-commands", "basics-reading-errors", "basics-a-small-program", "basics-where-next", "golden-split", "golden-phi-ratio", "golden-lucas", "golden-split-optimality", "golden-radix-economy", "golden-phi-in-gf16", "golden-gf4", "golden-gf6", "golden-gf8", "golden-gf10", "golden-gf12", "golden-gf14", "golden-gf16", "golden-dot-two", "golden-gf24", "golden-gf32", "golden-gf48", "golden-gf64", "golden-gf96", "golden-gf128", "golden-gf256", "golden-gf512", "golden-gf1024", "golden-gft8", "golden-gft16", "golden-gft32", "golden-decode", "t27-tri-vectors", "t27-tri-census", "t27-tri-seals", "t27-tri-prose", "t27-tri-lean", "tri-selftest", "play-terminal-gates", "tc-why-three", "tc-balanced-count", "tc-flip-and-cut", "tc-kleene-gates", "tc-consensus-any", "tc-sum-and-carry", "tc-half-adder", "tc-full-adder", "tc-ripple-carry", "tc-times-a-trit", "tc-long-multiply", "tc-ternary-mac", "tc-two-bits", "tc-five-in-a-byte", "tc-trit-word", "tc-instruction-word", "tc-opcode-map", "tc-immediate", "tc-ternary-alu", "tc-three-way-jump", "tc-run-a-program", "tc-compare", "tc-thirds", "tc-sort-three", "tc-one-neuron", "tc-detector", "tc-tiny-layer", "wars", "tri-game-tick", "tri-game-vault", "perf2-knockouts", "race-the-bee", "gatle", "tri-blog-list", "tri-game-selftest"];
-pub const WIDGET_KINDS : [252]str = ["cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "player", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "player", "player", "player", "cast", "cast", "tool", "tool", "cast", "player", "cast", "player", "cast", "player", "cast", "player", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tab", "cast", "cast", "cast", "tool", "tool", "cast", "cast"];
-pub const WIDGET_CATEGORIES : [252]str = ["fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "game", "game", "game", "game", "game", "game", "game", "game"];
-pub const WIDGET_PAGES : [252]str = ["term/x7-board/", "term/tri-x7-fasm/", "term/tri-x7-mutate/", "term/perf2-loop/", "term/tri-fpga-rgmii/", "term/tri-fpga-jtag/", "term/tri-fpga-pins/", "term/devkit-flow/", "play/hello-world/verilog/", "widgets/bit-diff/", "widgets/build-receipt/", "widgets/cost-curves/", "widgets/gf16-calc/", "widgets/idcode/", "widgets/lut-explain/", "widgets/pin-map/", "widgets/placement-map/", "widgets/resource-bars/", "widgets/slack-waterfall/", "widgets/test-waves/", "widgets/synth-smells/", "widgets/bit-autopsy/", "widgets/fasm-skyline/", "widgets/vcd-wrapped/", "widgets/fits-on/", "widgets/stat-card/", "widgets/lut-depth/", "widgets/fsm-sketch/", "widgets/lut-treemap/", "widgets/t27-cell/", "widgets/t27-vs-nvfp4/", "widgets/uart-bucket/", "term/tri-fpga-specs/", "term/tri-fpga-tools/", "term/tri-fpga-wire/", "term/tri-fpga-batch/", "term/tri-fpga-status/", "term/tri-fpga-tmpcheck/", "term/tri-fpga-keycheck/", "term/tri-fpga-selftest/", "term/tri-x7-refs/", "term/tri-x7-audit/", "term/t27c-clock-domain/", "term/t27c-clocks/", "term/t27c-wukong-v1/", "term/t27c-cts/", "term/t27c-mmcm/", "term/t27c-mmcm-mutate/", "term/t27c-timing/", "term/t27c-reset-sync/", "term/t27c-reset-sync-gen/", "term/t27c-mtbf/", "term/t27c-clock-domain-gen/", "term/t27c-fifo/", "term/t27c-fifo-mutate/", "term/t27c-fifo-flags/", "term/t27c-emitter-xdc/", "term/t27c-fpga-rxcap/", "term/t27-tri-competitors/", "term/t27-tri-discard/", "term/tri-fpga-txhold/", "term/tri-fpga-rxcap/", "term/tri-fpga-rxer/", "term/tri-fpga-steps/", "term/tri-fpga-ioclients/", "term/tri-fpga-claim/", "term/tri-fpga-release/", "term/tri-fpga-next/", "term/t27c-memory/", "term/t27c-spi-tb/", "term/t27c-check-uart/", "term/t27c-check-spi/", "term/t27c-check-apb/", "term/t27c-check-axi4/", "term/t27c-check-bridge/", "term/t27c-check-memory/", "term/t27c-verilog-uart/", "term/t27c-verilog-spi/", "term/t27c-verilog-apb/", "term/t27c-verilog-axi4/", "term/t27c-verilog-bridge/", "term/t27c-verilog-top/", "term/t27c-verilog-uart-tb/", "term/t27c-verilog-apb-tb/", "term/t27c-verilog-axi-tb/", "term/t27c-verilog-mem-tb/", "term/t27c-verilog-integration-tb/", "term/t27c-verilog-top-tb/", "term/t27c-verilog-spi-tb/", "term/t27c-hir-uart/", "term/t27c-hir-apb/", "term/t27c-hir-axi4/", "term/t27c-eth-crc/", "term/t27c-rgmii/", "widgets/jtag-verdict/", "widgets/line-bounce/", "widgets/eye-isi/", "widgets/ecc-secded/", "widgets/carry-chain/", "widgets/prefix-adder/", "widgets/csa-tree/", "widgets/shift-add/", "widgets/booth-radix4/", "widgets/dsp-fit/", "widgets/q-format/", "widgets/round-sat/", "widgets/sqnr/", "widgets/cordic/", "widgets/cordic-vector/", "widgets/sine-table/", "widgets/alias-fold/", "widgets/nco/", "widgets/dft-bin/", "widgets/fir-response/", "widgets/fir-design/", "widgets/fir-cost/", "widgets/decimator/", "widgets/cic-growth/", "widgets/polyphase/", "widgets/fft-butterfly/", "widgets/bit-reverse/", "widgets/fft-scaling/", "widgets/barker-corr/", "widgets/fir-fit/", "widgets/fir-on-board/", "play/hello-world/", "play/hello-world/rust/", "play/hello-world/js/", "term/t27c-hello-world/", "term/t27-tri-misread/", "widgets/test-touch/", "widgets/t27c-stages/", "term/t27c-e8m0/", "play/e8m0/", "term/t27c-ocp-mx/", "play/gfternary/", "term/t27c-gfternary/", "play/golden-sieve/", "term/t27c-golden-sieve/", "play/tnf17/", "term/t27c-tnf17/", "term/t27c-activation-quantizer/", "term/t27c-bitnet-majority/", "term/t27c-bitnet-neuron-nchunk/", "term/t27c-comb-ternary-dot/", "term/t27c-stream-ternary-mac/", "term/t27c-bitnet-mlp/", "term/t27c-gft-smul/", "term/t27c-gft-sadd/", "term/t27c-gft-signed-mac/", "term/t27c-gft-relu/", "term/t27c-gft-exp2/", "term/t27c-gft-argmax4/", "term/t27c-gft-nll/", "term/t27c-gft-sgd-step/", "term/t27c-gft-xornet/", "widgets/basics-what-a-spec-is/", "widgets/basics-the-module-line/", "widgets/basics-comments-and-prose/", "widgets/basics-constants/", "widgets/basics-whole-numbers/", "widgets/basics-true-false-and-text/", "widgets/basics-arrays/", "widgets/basics-trits/", "widgets/basics-expressions/", "widgets/basics-a-test-block/", "widgets/basics-many-tests/", "widgets/basics-functions/", "widgets/basics-local-names/", "widgets/basics-if-and-else/", "widgets/basics-loops/", "widgets/basics-pub-or-private/", "widgets/basics-structs/", "widgets/basics-enums/", "widgets/basics-use-other-modules/", "widgets/basics-invariants/", "widgets/basics-gen-ts/", "widgets/basics-seven-backends/", "widgets/basics-stages-of-t27c/", "widgets/basics-tri-commands/", "widgets/basics-reading-errors/", "widgets/basics-a-small-program/", "widgets/basics-where-next/", "widgets/golden-split/", "widgets/golden-phi-ratio/", "widgets/golden-lucas/", "widgets/golden-split-optimality/", "widgets/golden-radix-economy/", "widgets/golden-phi-in-gf16/", "widgets/golden-gf4/", "widgets/golden-gf6/", "widgets/golden-gf8/", "widgets/golden-gf10/", "widgets/golden-gf12/", "widgets/golden-gf14/", "widgets/golden-gf16/", "widgets/golden-dot-two/", "widgets/golden-gf24/", "widgets/golden-gf32/", "widgets/golden-gf48/", "widgets/golden-gf64/", "widgets/golden-gf96/", "widgets/golden-gf128/", "widgets/golden-gf256/", "widgets/golden-gf512/", "widgets/golden-gf1024/", "widgets/golden-gft8/", "widgets/golden-gft16/", "widgets/golden-gft32/", "widgets/golden-decode/", "term/t27-tri-vectors/", "term/t27-tri-census/", "term/t27-tri-seals/", "term/t27-tri-prose/", "term/t27-tri-lean/", "term/tri-selftest/", "term/play-terminal-gates/", "widgets/tc-why-three/", "widgets/tc-balanced-count/", "widgets/tc-flip-and-cut/", "widgets/tc-kleene-gates/", "widgets/tc-consensus-any/", "widgets/tc-sum-and-carry/", "widgets/tc-half-adder/", "widgets/tc-full-adder/", "widgets/tc-ripple-carry/", "widgets/tc-times-a-trit/", "widgets/tc-long-multiply/", "widgets/tc-ternary-mac/", "widgets/tc-two-bits/", "widgets/tc-five-in-a-byte/", "widgets/tc-trit-word/", "widgets/tc-instruction-word/", "widgets/tc-opcode-map/", "widgets/tc-immediate/", "widgets/tc-ternary-alu/", "widgets/tc-three-way-jump/", "widgets/tc-run-a-program/", "widgets/tc-compare/", "widgets/tc-thirds/", "widgets/tc-sort-three/", "widgets/tc-one-neuron/", "widgets/tc-detector/", "widgets/tc-tiny-layer/", "wars", "term/tri-game-tick/", "term/tri-game-vault/", "term/perf2-knockouts/", "widgets/race-the-bee/", "widgets/gatle/", "term/tri-blog-list/", "term/tri-game-selftest/"];
-pub const WIDGET_TITLES : [252]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "WARS: agent configurations compared on real GitHub issues", "", "", "", "", "", "", ""];
-pub const WIDGET_LINES : [252]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "One issue, one pinned commit, the same tools and gates; only the decision layer changes, and a missing run stays unknown.", "", "", "", "", "", "", ""];
+pub const WIDGET_COUNT : u16 = 279;
+pub const WIDGET_IDS : [279]str = ["x7-board", "tri-x7-fasm", "tri-x7-mutate", "perf2-loop", "tri-fpga-rgmii", "tri-fpga-jtag", "tri-fpga-pins", "devkit-flow", "play-verilog", "bit-diff", "build-receipt", "cost-curves", "gf16-calc", "idcode", "lut-explain", "pin-map", "placement-map", "resource-bars", "slack-waterfall", "test-waves", "synth-smells", "bit-autopsy", "fasm-skyline", "vcd-wrapped", "fits-on", "stat-card", "lut-depth", "fsm-sketch", "lut-treemap", "t27-cell", "t27-vs-nvfp4", "uart-bucket", "tri-fpga-specs", "tri-fpga-tools", "tri-fpga-wire", "tri-fpga-batch", "tri-fpga-status", "tri-fpga-tmpcheck", "tri-fpga-keycheck", "tri-fpga-selftest", "tri-x7-refs", "tri-x7-audit", "t27c-clock-domain", "t27c-clocks", "t27c-wukong-v1", "t27c-cts", "t27c-mmcm", "t27c-mmcm-mutate", "t27c-timing", "t27c-reset-sync", "t27c-reset-sync-gen", "t27c-mtbf", "t27c-clock-domain-gen", "t27c-fifo", "t27c-fifo-mutate", "t27c-fifo-flags", "t27c-emitter-xdc", "t27c-fpga-rxcap", "t27-tri-competitors", "t27-tri-discard", "tri-fpga-txhold", "tri-fpga-rxcap", "tri-fpga-rxer", "tri-fpga-steps", "tri-fpga-ioclients", "tri-fpga-claim", "tri-fpga-release", "tri-fpga-next", "t27c-memory", "t27c-spi-tb", "t27c-check-uart", "t27c-check-spi", "t27c-check-apb", "t27c-check-axi4", "t27c-check-bridge", "t27c-check-memory", "t27c-verilog-uart", "t27c-verilog-spi", "t27c-verilog-apb", "t27c-verilog-axi4", "t27c-verilog-bridge", "t27c-verilog-top", "t27c-verilog-uart-tb", "t27c-verilog-apb-tb", "t27c-verilog-axi-tb", "t27c-verilog-mem-tb", "t27c-verilog-integration-tb", "t27c-verilog-top-tb", "t27c-verilog-spi-tb", "t27c-hir-uart", "t27c-hir-apb", "t27c-hir-axi4", "t27c-eth-crc", "t27c-rgmii", "jtag-verdict", "line-bounce", "eye-isi", "ecc-secded", "carry-chain", "prefix-adder", "csa-tree", "shift-add", "booth-radix4", "dsp-fit", "q-format", "round-sat", "sqnr", "cordic", "cordic-vector", "sine-table", "alias-fold", "nco", "dft-bin", "fir-response", "fir-design", "fir-cost", "decimator", "cic-growth", "polyphase", "fft-butterfly", "bit-reverse", "fft-scaling", "barker-corr", "fir-fit", "fir-on-board", "play-hello-world", "play-rust", "play-js", "t27c-hello-world", "t27-tri-misread", "test-touch", "t27c-stages", "t27c-e8m0", "play-e8m0", "t27c-ocp-mx", "play-gfternary", "t27c-gfternary", "play-golden-sieve", "t27c-golden-sieve", "play-tnf17", "t27c-tnf17", "t27c-activation-quantizer", "t27c-bitnet-majority", "t27c-bitnet-neuron-nchunk", "t27c-comb-ternary-dot", "t27c-stream-ternary-mac", "t27c-bitnet-mlp", "t27c-gft-smul", "t27c-gft-sadd", "t27c-gft-signed-mac", "t27c-gft-relu", "t27c-gft-exp2", "t27c-gft-argmax4", "t27c-gft-nll", "t27c-gft-sgd-step", "t27c-gft-xornet", "basics-what-a-spec-is", "basics-the-module-line", "basics-comments-and-prose", "basics-constants", "basics-whole-numbers", "basics-true-false-and-text", "basics-arrays", "basics-trits", "basics-expressions", "basics-a-test-block", "basics-many-tests", "basics-functions", "basics-local-names", "basics-if-and-else", "basics-loops", "basics-pub-or-private", "basics-structs", "basics-enums", "basics-use-other-modules", "basics-invariants", "basics-gen-ts", "basics-seven-backends", "basics-stages-of-t27c", "basics-tri-commands", "basics-reading-errors", "basics-a-small-program", "basics-where-next", "golden-split", "golden-phi-ratio", "golden-lucas", "golden-split-optimality", "golden-radix-economy", "golden-phi-in-gf16", "golden-gf4", "golden-gf6", "golden-gf8", "golden-gf10", "golden-gf12", "golden-gf14", "golden-gf16", "golden-dot-two", "golden-gf24", "golden-gf32", "golden-gf48", "golden-gf64", "golden-gf96", "golden-gf128", "golden-gf256", "golden-gf512", "golden-gf1024", "golden-gft8", "golden-gft16", "golden-gft32", "golden-decode", "t27-tri-vectors", "t27-tri-census", "t27-tri-seals", "t27-tri-prose", "t27-tri-lean", "tri-selftest", "play-terminal-gates", "tc-why-three", "tc-balanced-count", "tc-flip-and-cut", "tc-kleene-gates", "tc-consensus-any", "tc-sum-and-carry", "tc-half-adder", "tc-full-adder", "tc-ripple-carry", "tc-times-a-trit", "tc-long-multiply", "tc-ternary-mac", "tc-two-bits", "tc-five-in-a-byte", "tc-trit-word", "tc-instruction-word", "tc-opcode-map", "tc-immediate", "tc-ternary-alu", "tc-three-way-jump", "tc-run-a-program", "tc-compare", "tc-thirds", "tc-sort-three", "tc-one-neuron", "tc-detector", "tc-tiny-layer", "nn-relu", "nn-sigmoid", "nn-tanh", "nn-gelu", "nn-silu", "nn-elu", "nn-softmax", "nn-embedding", "nn-dropout", "nn-conv", "nn-maxpool", "nn-batchnorm", "nn-mse", "nn-bce", "nn-huber", "nn-sgd", "nn-rmsprop", "nn-adam", "nn-backprop", "nn-xor", "nn-lamb", "nn-rnn", "nn-lstm", "nn-attention", "nn-positions", "nn-encoder", "nn-ppo", "wars", "tri-game-tick", "tri-game-vault", "perf2-knockouts", "race-the-bee", "gatle", "tri-blog-list", "tri-game-selftest"];
+pub const WIDGET_KINDS : [279]str = ["cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "player", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "player", "player", "player", "cast", "cast", "tool", "tool", "cast", "player", "cast", "player", "cast", "player", "cast", "player", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "cast", "cast", "cast", "cast", "cast", "cast", "cast", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tool", "tab", "cast", "cast", "cast", "tool", "tool", "cast", "cast"];
+pub const WIDGET_CATEGORIES : [279]str = ["fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "fpga", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "compiler", "game", "game", "game", "game", "game", "game", "game", "game"];
+pub const WIDGET_PAGES : [279]str = ["term/x7-board/", "term/tri-x7-fasm/", "term/tri-x7-mutate/", "term/perf2-loop/", "term/tri-fpga-rgmii/", "term/tri-fpga-jtag/", "term/tri-fpga-pins/", "term/devkit-flow/", "play/hello-world/verilog/", "widgets/bit-diff/", "widgets/build-receipt/", "widgets/cost-curves/", "widgets/gf16-calc/", "widgets/idcode/", "widgets/lut-explain/", "widgets/pin-map/", "widgets/placement-map/", "widgets/resource-bars/", "widgets/slack-waterfall/", "widgets/test-waves/", "widgets/synth-smells/", "widgets/bit-autopsy/", "widgets/fasm-skyline/", "widgets/vcd-wrapped/", "widgets/fits-on/", "widgets/stat-card/", "widgets/lut-depth/", "widgets/fsm-sketch/", "widgets/lut-treemap/", "widgets/t27-cell/", "widgets/t27-vs-nvfp4/", "widgets/uart-bucket/", "term/tri-fpga-specs/", "term/tri-fpga-tools/", "term/tri-fpga-wire/", "term/tri-fpga-batch/", "term/tri-fpga-status/", "term/tri-fpga-tmpcheck/", "term/tri-fpga-keycheck/", "term/tri-fpga-selftest/", "term/tri-x7-refs/", "term/tri-x7-audit/", "term/t27c-clock-domain/", "term/t27c-clocks/", "term/t27c-wukong-v1/", "term/t27c-cts/", "term/t27c-mmcm/", "term/t27c-mmcm-mutate/", "term/t27c-timing/", "term/t27c-reset-sync/", "term/t27c-reset-sync-gen/", "term/t27c-mtbf/", "term/t27c-clock-domain-gen/", "term/t27c-fifo/", "term/t27c-fifo-mutate/", "term/t27c-fifo-flags/", "term/t27c-emitter-xdc/", "term/t27c-fpga-rxcap/", "term/t27-tri-competitors/", "term/t27-tri-discard/", "term/tri-fpga-txhold/", "term/tri-fpga-rxcap/", "term/tri-fpga-rxer/", "term/tri-fpga-steps/", "term/tri-fpga-ioclients/", "term/tri-fpga-claim/", "term/tri-fpga-release/", "term/tri-fpga-next/", "term/t27c-memory/", "term/t27c-spi-tb/", "term/t27c-check-uart/", "term/t27c-check-spi/", "term/t27c-check-apb/", "term/t27c-check-axi4/", "term/t27c-check-bridge/", "term/t27c-check-memory/", "term/t27c-verilog-uart/", "term/t27c-verilog-spi/", "term/t27c-verilog-apb/", "term/t27c-verilog-axi4/", "term/t27c-verilog-bridge/", "term/t27c-verilog-top/", "term/t27c-verilog-uart-tb/", "term/t27c-verilog-apb-tb/", "term/t27c-verilog-axi-tb/", "term/t27c-verilog-mem-tb/", "term/t27c-verilog-integration-tb/", "term/t27c-verilog-top-tb/", "term/t27c-verilog-spi-tb/", "term/t27c-hir-uart/", "term/t27c-hir-apb/", "term/t27c-hir-axi4/", "term/t27c-eth-crc/", "term/t27c-rgmii/", "widgets/jtag-verdict/", "widgets/line-bounce/", "widgets/eye-isi/", "widgets/ecc-secded/", "widgets/carry-chain/", "widgets/prefix-adder/", "widgets/csa-tree/", "widgets/shift-add/", "widgets/booth-radix4/", "widgets/dsp-fit/", "widgets/q-format/", "widgets/round-sat/", "widgets/sqnr/", "widgets/cordic/", "widgets/cordic-vector/", "widgets/sine-table/", "widgets/alias-fold/", "widgets/nco/", "widgets/dft-bin/", "widgets/fir-response/", "widgets/fir-design/", "widgets/fir-cost/", "widgets/decimator/", "widgets/cic-growth/", "widgets/polyphase/", "widgets/fft-butterfly/", "widgets/bit-reverse/", "widgets/fft-scaling/", "widgets/barker-corr/", "widgets/fir-fit/", "widgets/fir-on-board/", "play/hello-world/", "play/hello-world/rust/", "play/hello-world/js/", "term/t27c-hello-world/", "term/t27-tri-misread/", "widgets/test-touch/", "widgets/t27c-stages/", "term/t27c-e8m0/", "play/e8m0/", "term/t27c-ocp-mx/", "play/gfternary/", "term/t27c-gfternary/", "play/golden-sieve/", "term/t27c-golden-sieve/", "play/tnf17/", "term/t27c-tnf17/", "term/t27c-activation-quantizer/", "term/t27c-bitnet-majority/", "term/t27c-bitnet-neuron-nchunk/", "term/t27c-comb-ternary-dot/", "term/t27c-stream-ternary-mac/", "term/t27c-bitnet-mlp/", "term/t27c-gft-smul/", "term/t27c-gft-sadd/", "term/t27c-gft-signed-mac/", "term/t27c-gft-relu/", "term/t27c-gft-exp2/", "term/t27c-gft-argmax4/", "term/t27c-gft-nll/", "term/t27c-gft-sgd-step/", "term/t27c-gft-xornet/", "widgets/basics-what-a-spec-is/", "widgets/basics-the-module-line/", "widgets/basics-comments-and-prose/", "widgets/basics-constants/", "widgets/basics-whole-numbers/", "widgets/basics-true-false-and-text/", "widgets/basics-arrays/", "widgets/basics-trits/", "widgets/basics-expressions/", "widgets/basics-a-test-block/", "widgets/basics-many-tests/", "widgets/basics-functions/", "widgets/basics-local-names/", "widgets/basics-if-and-else/", "widgets/basics-loops/", "widgets/basics-pub-or-private/", "widgets/basics-structs/", "widgets/basics-enums/", "widgets/basics-use-other-modules/", "widgets/basics-invariants/", "widgets/basics-gen-ts/", "widgets/basics-seven-backends/", "widgets/basics-stages-of-t27c/", "widgets/basics-tri-commands/", "widgets/basics-reading-errors/", "widgets/basics-a-small-program/", "widgets/basics-where-next/", "widgets/golden-split/", "widgets/golden-phi-ratio/", "widgets/golden-lucas/", "widgets/golden-split-optimality/", "widgets/golden-radix-economy/", "widgets/golden-phi-in-gf16/", "widgets/golden-gf4/", "widgets/golden-gf6/", "widgets/golden-gf8/", "widgets/golden-gf10/", "widgets/golden-gf12/", "widgets/golden-gf14/", "widgets/golden-gf16/", "widgets/golden-dot-two/", "widgets/golden-gf24/", "widgets/golden-gf32/", "widgets/golden-gf48/", "widgets/golden-gf64/", "widgets/golden-gf96/", "widgets/golden-gf128/", "widgets/golden-gf256/", "widgets/golden-gf512/", "widgets/golden-gf1024/", "widgets/golden-gft8/", "widgets/golden-gft16/", "widgets/golden-gft32/", "widgets/golden-decode/", "term/t27-tri-vectors/", "term/t27-tri-census/", "term/t27-tri-seals/", "term/t27-tri-prose/", "term/t27-tri-lean/", "term/tri-selftest/", "term/play-terminal-gates/", "widgets/tc-why-three/", "widgets/tc-balanced-count/", "widgets/tc-flip-and-cut/", "widgets/tc-kleene-gates/", "widgets/tc-consensus-any/", "widgets/tc-sum-and-carry/", "widgets/tc-half-adder/", "widgets/tc-full-adder/", "widgets/tc-ripple-carry/", "widgets/tc-times-a-trit/", "widgets/tc-long-multiply/", "widgets/tc-ternary-mac/", "widgets/tc-two-bits/", "widgets/tc-five-in-a-byte/", "widgets/tc-trit-word/", "widgets/tc-instruction-word/", "widgets/tc-opcode-map/", "widgets/tc-immediate/", "widgets/tc-ternary-alu/", "widgets/tc-three-way-jump/", "widgets/tc-run-a-program/", "widgets/tc-compare/", "widgets/tc-thirds/", "widgets/tc-sort-three/", "widgets/tc-one-neuron/", "widgets/tc-detector/", "widgets/tc-tiny-layer/", "widgets/nn-relu/", "widgets/nn-sigmoid/", "widgets/nn-tanh/", "widgets/nn-gelu/", "widgets/nn-silu/", "widgets/nn-elu/", "widgets/nn-softmax/", "widgets/nn-embedding/", "widgets/nn-dropout/", "widgets/nn-conv/", "widgets/nn-maxpool/", "widgets/nn-batchnorm/", "widgets/nn-mse/", "widgets/nn-bce/", "widgets/nn-huber/", "widgets/nn-sgd/", "widgets/nn-rmsprop/", "widgets/nn-adam/", "widgets/nn-backprop/", "widgets/nn-xor/", "widgets/nn-lamb/", "widgets/nn-rnn/", "widgets/nn-lstm/", "widgets/nn-attention/", "widgets/nn-positions/", "widgets/nn-encoder/", "widgets/nn-ppo/", "wars", "term/tri-game-tick/", "term/tri-game-vault/", "term/perf2-knockouts/", "widgets/race-the-bee/", "widgets/gatle/", "term/tri-blog-list/", "term/tri-game-selftest/"];
+pub const WIDGET_TITLES : [279]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "WARS: agent configurations compared on real GitHub issues", "", "", "", "", "", "", ""];
+pub const WIDGET_LINES : [279]str = ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "One issue, one pinned commit, the same tools and gates; only the decision layer changes, and a missing run stays unknown.", "", "", "", "", "", "", ""];
; Why a person would pass this one on: the reason the widget is in the gallery at all.
-pub const WIDGET_HOOKS : [252]str = ["The bitstream is byte-identical to openXC7's and runs on a real board: the claim most FPGA people doubt, shown.", "A Xilinx 7-series bitstream assembled with no Vivado anywhere in the path.", "Plant a bug in the bitstream path and watch the tests catch it: mutation testing for hardware.", "Place and route to a loaded bitstream, with the clock on screen.", "RGMII hold margins read from nextpnr's own SDF, not from a datasheet.", "The cable asks which chip is there and the chip answers with its IDCODE.", "Before an edit to a Verilog file: every spec and generator it would touch.", "The whole FPGA flow, one layer per line, from spec to board.", "A spec turned into Verilog inside a post, with the RTL read the way a hardware engineer reads it.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "17 of 17 conformance specs compile, their test blocks pass, and no generated file has drifted.", "14 of 14 bench tools pass their own self-tests in one sweep: decode 18/18, repin 17/17, firelog 10/10, tmpcheck 9/9, stamps 9/9, keep 9/9.", "Requested 115,200 puts 115,385 on the wire (+0.16 %), 921,600 puts 923,077, and 3,000,000 is exact.", "26 checks, no board, no RTL: every receipt returns, 11 loss and fault cases recover, and a lie stops the run.", "One command reads the bench: cables, locks, and the last board runs: three x7-board runs PASS 51840/51840.", "112 files under /tmp pinned by sha256 in specs: every one has a kept copy, 0 lost, 0 unlocated.", "84 board and model logs scanned for the TRI-NET key: 0 hits, 0 logs of 1 MB or more.", "16 spec-named scripts run their own --self-test, 0 bad; 8 without the flag are listed, not skipped silently.", "The part table in far.t27 matches prjxray-db part.json, all 6 reference bitstreams present, bitwalk reads all 6 in 84.4 ms.", "One command re-runs every sweep: CRC 12/12, ECC 32520/32520 frames, 82/82 pins, 4224/4224 segbits, gold 5/5. audit: CLEAN.", "The CDC spec through the real compiler: 12 tests pass natively and the header says what the model does and does not claim.", "The tri clocks card as a spec: constants a tool page reads, and a native test-report that says so plainly -- 0 tests, because it is not a test suite.", "The board itself as a spec: 15 tests pass natively and the part line names the xc7a200tfgg676 the Wukong carries.", "Clock-tree synthesis in numbers: 15 tests pass natively, and the header pins the Artix-7 teaching delays the tree arithmetic uses.", "The MMCM model through the real compiler: VCO window, M/D/O ranges, fractional multiply in eighths, the 1/56 phase step, 16 tests natively.", "Widen the VCO window with one sed and exactly one test objects -- the rejection test that names the 500 MHz config it calls out.", "Static timing as arithmetic: 25 tests pass natively on arcs, slack and Fmax, with the Artix-7 teaching delays in the header.", "The three reset kinds through the real compiler: async assert with sync release through 2 to 4 flops, and the racy async release, 16 tests natively.", "What the compiler emits for the reset pipe: t27c gen prints the Zig, one struct copy per clock, and wc -l counts it.", "Metastability MTBF without floats: the WP323 form in Q10 log2, one flop negative slack, two flops astronomical, 15 tests natively.", "The CDC model as Zig: t27c gen prints what it compiled and wc -l says how many lines of code the model is.", "The FIFO stdlib through the real compiler: 18 tests pass natively and 6 invariants are proved comptime.", "Move a pointer by 3 instead of 1 and every test still passes: the suite asserts fill and flags, never pointer values.", "Flip one flag assignment and a single test objects: the guard that keeps the FIFO honest.", "The pins-to-XDC emitter spec parsed by the real compiler, and the create_clock lines it exists to write.", "The fpga-rxcap tool card as a spec: constants for a tool page, and a native run that says 0 tests and why.", "168 records: 14 papers entered twice, 141 cite no score, 144 state zero at pass@1.", "67 specs lose 23,578 tokens to parser recovery, each held at its pinned bound.", "Smallest hold 3445 ps on TXD1 against a 1000 ps bench; each TX pin setup and hold from the routed SDF.", "Per-bit setup and hold of an async RX nibble against its RXC edge, from the routed SDF: one nibble changes 2387 to 2486 ps apart, so a binary counter tears where a gray one cannot.", "RX_ER pulses injected at exact nibble positions of a ping -- preamble, mid-frame, IP header -- and the frame state each pulse leaves behind, RTL vs model.", "The Ethernet bring-up plan on the bench: pre-registered steps, logged exits, what ran and what did not.", "The registered IO clients on the bench, read live: names, claims and what each declares it owns.", "The bench claim in action: one client takes the hardware lock and the tool reports who holds it.", "The bench lock given back: the client gives up its claim and the bench reports itself free.", "The next queued step on the bench, read live from the run state.", "The memory map spec through the real compiler: 15 tests pass natively and 6 invariants are proved comptime.", "The SPI testbench through the real compiler: 7 tests pass natively, driving the modes and checking the captures.", "t27c check on the UART spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the SPI spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the APB bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the AXI4 spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the bus bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the memory spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "The UART spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The SPI spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The APB bridge spec lowered to synthesizable Verilog: port list, PSEL and PENABLE wiring, the address decode.", "The AXI4 spec lowered to synthesizable Verilog: the five channels' wiring and handshakes as generated wires.", "The bus bridge spec lowered to synthesizable Verilog: both sides of the crossing as generated wires.", "The whole design in one spec lowered to synthesizable Verilog: clocks, resets and bus wiring as the compiler emits them.", "The UART testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The APB bridge testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The AXI4 testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The memory testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The integration testbench spec lowered to Verilog: every bus of the design in one bench.", "The top-level testbench spec lowered to Verilog: the whole design under one bench.", "The SPI testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The UART spec lowered to hardware IR: the compiler's own view of the wires and state before Verilog exists.", "The APB bridge spec lowered to hardware IR: PSEL, PENABLE and the decode as the compiler structures them.", "The AXI4 spec lowered to hardware IR: the five channels and their handshakes as the compiler structures them.", "The Ethernet FCS spec through the real compiler: 8 tests pass natively -- the table derived from one bit step, the 0xCBF43926 check value, the 0xDEBB20E3 receiver residue.", "The RGMII spec through the real compiler: 7 tests pass natively and 1 invariant is proved comptime -- the 8 ns gigabit period, a nibble on both edges, the 78 ps tap ladder.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "The t27 compiler running as WebAssembly in a post: compile, test, seven backends.", "The same spec as Rust, printed by a terminal in the page.", "The only backend that runs in the browser: generate the module, then run it.", "t27c on hello_world.t27, on a real machine, every byte as printed.", "Code that compiles and is wrong: the tool that finds it.", "", "", "The OCP MX scale byte as a spec: 2^(e-127), code 255 is NaN, 18 tests and 9 comptime invariants, compiled on a real machine.", "The same scale byte compiled in your browser: seven backends, the spec's tests run in the page, and every skip says why.", "One outlier sets the scale for 32 weights: MXFP4 flushes 23 of them to zero, MXFP8 E4M3 none. 49 tests pass; a planted bug fails one.", "Ternary weights scaled by phi, compiled in your browser: codes 00, 01, 10, and the fourth code folded to zero.", "13 of 13 tests pass natively, and the report says 8 of them executed no assert; a planted bug fails exactly one.", "Five measured rules for a ternary weight alphabet, and the one formula that passes all five, run in your browser.", "One more trit and the spec does not compile at all: BLOCKED, not failed, because no test ever ran.", "A 17-bit float whose exponent is four balanced trits, 3^4 = 81 values, compiled in your browser.", "34 tests pass natively, none vacuous; swap one XOR for OR and only the test that negates twice fails.", "A neuron's sum squeezed back to one trit: 7 tests pass natively, and turning one > into >= fails only the boundary test.", "One ternary neuron, two functions picked by its weights: 12 tests pass natively, and one weight moved from P to Z fails exactly one.", "A neuron wider than 27 inputs, summed chunk by chunk: 11 tests pass natively, and a loop that stops after one chunk fails exactly one.", "A 27-lane ternary dot product as pure wires: 4 tests pass natively, and one wrong sign in tmul fails exactly one.", "The same dot product added into a register on every clock: 4 tests pass natively, and a lane mask short one bit fails exactly one.", "A whole small ternary network, 3 neurons then 2: 4 tests pass natively, and a trit packed one bit low fails exactly one.", "A signed multiply: 3 tests pass natively, none vacuous, and a sign stuck at 0 fails exactly one, m2.", "A signed add: 3 tests pass natively, and adding where opposite signs must subtract fails exactly one, a2.", "Two signed products summed: 4 tests pass natively, and OR in place of the sign XOR fails exactly one, pp.", "A ReLU: 4 tests pass natively, and letting negative inputs through fails only negz, the one test with a negative input.", "2^x for a softmax: 4 tests pass natively, and dropping the minus sign of k fails exactly one, em1.", "argmax by comparing bits alone: 4 tests pass natively, and >= in place of > fails exactly one, tie_low.", "A loss in bits: 3 tests pass natively, and a zero with the sign bit set fails exactly one, perfect.", "One SGD step: 3 tests pass natively, and g times g in place of eta times g fails exactly one, ascend.", "A 2-layer ReLU net for XOR: 4 tests pass natively, and one relu removed fails exactly one, x00.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "34 vector files, 512 cases: 2 files executed, 8 debt, 24 prose-only; 19% carry data.", "Four census populations recounted another way all agree; three are named as uncheckable.", "31 of 1,428 seals record no output (16 specs); 3 of those are missing from the debt ledger.", "11 specs refused at parse, 0 blocked only by prose, 23 broken on purpose as fixtures.", "12 Lean files, 15,553 lines and 4 sorry, are reached by no build root: nothing compiles them.", "tri, railway, python3 and the loop directory: each present or not, in one call.", "The terminal for seven t27 backends: every command run on hello_world. 21 pass, 0 fail.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "Pick a lane and see whether the TRI decision layer beats the baseline on the same issue.", "One tick of the Queen's loop: an anomaly scan that reports only what changed.", "The loop's skills checked against snapshots before an agent may use them.", "Three performance bits, and one knockout bitstream per bit to prove each one matters.", "", "", "73 published posts with date, language and revision, newest first.", "6 of 6 self-tests hold: anomalies, blog_drift, lessons, site_gate, tick, vault."];
+pub const WIDGET_HOOKS : [279]str = ["The bitstream is byte-identical to openXC7's and runs on a real board: the claim most FPGA people doubt, shown.", "A Xilinx 7-series bitstream assembled with no Vivado anywhere in the path.", "Plant a bug in the bitstream path and watch the tests catch it: mutation testing for hardware.", "Place and route to a loaded bitstream, with the clock on screen.", "RGMII hold margins read from nextpnr's own SDF, not from a datasheet.", "The cable asks which chip is there and the chip answers with its IDCODE.", "Before an edit to a Verilog file: every spec and generator it would touch.", "The whole FPGA flow, one layer per line, from spec to board.", "A spec turned into Verilog inside a post, with the RTL read the way a hardware engineer reads it.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "17 of 17 conformance specs compile, their test blocks pass, and no generated file has drifted.", "14 of 14 bench tools pass their own self-tests in one sweep: decode 18/18, repin 17/17, firelog 10/10, tmpcheck 9/9, stamps 9/9, keep 9/9.", "Requested 115,200 puts 115,385 on the wire (+0.16 %), 921,600 puts 923,077, and 3,000,000 is exact.", "26 checks, no board, no RTL: every receipt returns, 11 loss and fault cases recover, and a lie stops the run.", "One command reads the bench: cables, locks, and the last board runs: three x7-board runs PASS 51840/51840.", "112 files under /tmp pinned by sha256 in specs: every one has a kept copy, 0 lost, 0 unlocated.", "84 board and model logs scanned for the TRI-NET key: 0 hits, 0 logs of 1 MB or more.", "16 spec-named scripts run their own --self-test, 0 bad; 8 without the flag are listed, not skipped silently.", "The part table in far.t27 matches prjxray-db part.json, all 6 reference bitstreams present, bitwalk reads all 6 in 84.4 ms.", "One command re-runs every sweep: CRC 12/12, ECC 32520/32520 frames, 82/82 pins, 4224/4224 segbits, gold 5/5. audit: CLEAN.", "The CDC spec through the real compiler: 12 tests pass natively and the header says what the model does and does not claim.", "The tri clocks card as a spec: constants a tool page reads, and a native test-report that says so plainly -- 0 tests, because it is not a test suite.", "The board itself as a spec: 15 tests pass natively and the part line names the xc7a200tfgg676 the Wukong carries.", "Clock-tree synthesis in numbers: 15 tests pass natively, and the header pins the Artix-7 teaching delays the tree arithmetic uses.", "The MMCM model through the real compiler: VCO window, M/D/O ranges, fractional multiply in eighths, the 1/56 phase step, 16 tests natively.", "Widen the VCO window with one sed and exactly one test objects -- the rejection test that names the 500 MHz config it calls out.", "Static timing as arithmetic: 25 tests pass natively on arcs, slack and Fmax, with the Artix-7 teaching delays in the header.", "The three reset kinds through the real compiler: async assert with sync release through 2 to 4 flops, and the racy async release, 16 tests natively.", "What the compiler emits for the reset pipe: t27c gen prints the Zig, one struct copy per clock, and wc -l counts it.", "Metastability MTBF without floats: the WP323 form in Q10 log2, one flop negative slack, two flops astronomical, 15 tests natively.", "The CDC model as Zig: t27c gen prints what it compiled and wc -l says how many lines of code the model is.", "The FIFO stdlib through the real compiler: 18 tests pass natively and 6 invariants are proved comptime.", "Move a pointer by 3 instead of 1 and every test still passes: the suite asserts fill and flags, never pointer values.", "Flip one flag assignment and a single test objects: the guard that keeps the FIFO honest.", "The pins-to-XDC emitter spec parsed by the real compiler, and the create_clock lines it exists to write.", "The fpga-rxcap tool card as a spec: constants for a tool page, and a native run that says 0 tests and why.", "168 records: 14 papers entered twice, 141 cite no score, 144 state zero at pass@1.", "67 specs lose 23,578 tokens to parser recovery, each held at its pinned bound.", "Smallest hold 3445 ps on TXD1 against a 1000 ps bench; each TX pin setup and hold from the routed SDF.", "Per-bit setup and hold of an async RX nibble against its RXC edge, from the routed SDF: one nibble changes 2387 to 2486 ps apart, so a binary counter tears where a gray one cannot.", "RX_ER pulses injected at exact nibble positions of a ping -- preamble, mid-frame, IP header -- and the frame state each pulse leaves behind, RTL vs model.", "The Ethernet bring-up plan on the bench: pre-registered steps, logged exits, what ran and what did not.", "The registered IO clients on the bench, read live: names, claims and what each declares it owns.", "The bench claim in action: one client takes the hardware lock and the tool reports who holds it.", "The bench lock given back: the client gives up its claim and the bench reports itself free.", "The next queued step on the bench, read live from the run state.", "The memory map spec through the real compiler: 15 tests pass natively and 6 invariants are proved comptime.", "The SPI testbench through the real compiler: 7 tests pass natively, driving the modes and checking the captures.", "t27c check on the UART spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the SPI spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the APB bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the AXI4 spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the bus bridge spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "t27c check on the memory spec: 0 errors, 0 warnings -- the first bar every lesson's spec clears.", "The UART spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The SPI spec lowered to synthesizable Verilog: the TRINITY banner, the module port list and the datapath as wires.", "The APB bridge spec lowered to synthesizable Verilog: port list, PSEL and PENABLE wiring, the address decode.", "The AXI4 spec lowered to synthesizable Verilog: the five channels' wiring and handshakes as generated wires.", "The bus bridge spec lowered to synthesizable Verilog: both sides of the crossing as generated wires.", "The whole design in one spec lowered to synthesizable Verilog: clocks, resets and bus wiring as the compiler emits them.", "The UART testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The APB bridge testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The AXI4 testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The memory testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The integration testbench spec lowered to Verilog: every bus of the design in one bench.", "The top-level testbench spec lowered to Verilog: the whole design under one bench.", "The SPI testbench spec lowered to Verilog: what a bench looks like when the compiler emits it.", "The UART spec lowered to hardware IR: the compiler's own view of the wires and state before Verilog exists.", "The APB bridge spec lowered to hardware IR: PSEL, PENABLE and the decode as the compiler structures them.", "The AXI4 spec lowered to hardware IR: the five channels and their handshakes as the compiler structures them.", "The Ethernet FCS spec through the real compiler: 8 tests pass natively -- the table derived from one bit step, the 0xCBF43926 check value, the 0xDEBB20E3 receiver residue.", "The RGMII spec through the real compiler: 7 tests pass natively and 1 invariant is proved comptime -- the 8 ns gigabit period, a nibble on both edges, the 78 ps tap ladder.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "The t27 compiler running as WebAssembly in a post: compile, test, seven backends.", "The same spec as Rust, printed by a terminal in the page.", "The only backend that runs in the browser: generate the module, then run it.", "t27c on hello_world.t27, on a real machine, every byte as printed.", "Code that compiles and is wrong: the tool that finds it.", "", "", "The OCP MX scale byte as a spec: 2^(e-127), code 255 is NaN, 18 tests and 9 comptime invariants, compiled on a real machine.", "The same scale byte compiled in your browser: seven backends, the spec's tests run in the page, and every skip says why.", "One outlier sets the scale for 32 weights: MXFP4 flushes 23 of them to zero, MXFP8 E4M3 none. 49 tests pass; a planted bug fails one.", "Ternary weights scaled by phi, compiled in your browser: codes 00, 01, 10, and the fourth code folded to zero.", "13 of 13 tests pass natively, and the report says 8 of them executed no assert; a planted bug fails exactly one.", "Five measured rules for a ternary weight alphabet, and the one formula that passes all five, run in your browser.", "One more trit and the spec does not compile at all: BLOCKED, not failed, because no test ever ran.", "A 17-bit float whose exponent is four balanced trits, 3^4 = 81 values, compiled in your browser.", "34 tests pass natively, none vacuous; swap one XOR for OR and only the test that negates twice fails.", "A neuron's sum squeezed back to one trit: 7 tests pass natively, and turning one > into >= fails only the boundary test.", "One ternary neuron, two functions picked by its weights: 12 tests pass natively, and one weight moved from P to Z fails exactly one.", "A neuron wider than 27 inputs, summed chunk by chunk: 11 tests pass natively, and a loop that stops after one chunk fails exactly one.", "A 27-lane ternary dot product as pure wires: 4 tests pass natively, and one wrong sign in tmul fails exactly one.", "The same dot product added into a register on every clock: 4 tests pass natively, and a lane mask short one bit fails exactly one.", "A whole small ternary network, 3 neurons then 2: 4 tests pass natively, and a trit packed one bit low fails exactly one.", "A signed multiply: 3 tests pass natively, none vacuous, and a sign stuck at 0 fails exactly one, m2.", "A signed add: 3 tests pass natively, and adding where opposite signs must subtract fails exactly one, a2.", "Two signed products summed: 4 tests pass natively, and OR in place of the sign XOR fails exactly one, pp.", "A ReLU: 4 tests pass natively, and letting negative inputs through fails only negz, the one test with a negative input.", "2^x for a softmax: 4 tests pass natively, and dropping the minus sign of k fails exactly one, em1.", "argmax by comparing bits alone: 4 tests pass natively, and >= in place of > fails exactly one, tie_low.", "A loss in bits: 3 tests pass natively, and a zero with the sign bit set fails exactly one, perfect.", "One SGD step: 3 tests pass natively, and g times g in place of eta times g fails exactly one, ascend.", "A 2-layer ReLU net for XOR: 4 tests pass natively, and one relu removed fails exactly one, x00.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "34 vector files, 512 cases: 2 files executed, 8 debt, 24 prose-only; 19% carry data.", "Four census populations recounted another way all agree; three are named as uncheckable.", "31 of 1,428 seals record no output (16 specs); 3 of those are missing from the debt ledger.", "11 specs refused at parse, 0 blocked only by prose, 23 broken on purpose as fixtures.", "12 Lean files, 15,553 lines and 4 sorry, are reached by no build root: nothing compiles them.", "tri, railway, python3 and the loop directory: each present or not, in one call.", "The terminal for seven t27 backends: every command run on hello_world. 21 pass, 0 fail.", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "Pick a lane and see whether the TRI decision layer beats the baseline on the same issue.", "One tick of the Queen's loop: an anomaly scan that reports only what changed.", "The loop's skills checked against snapshots before an agent may use them.", "Three performance bits, and one knockout bitstream per bit to prove each one matters.", "", "", "73 published posts with date, language and revision, newest first.", "6 of 6 self-tests hold: anomalies, blog_drift, lessons, site_gate, tick, vault."];
; --- Ideas: asked for, not built ---------------------------------------------------------------
; From what engineers share today (gh-skyline, Tiny Tapeout renders, Wrapped cards, Gatle-style
@@ -119,9 +119,9 @@ test the_fpga_shelf_comes_first_and_opens_on_the_board {
}
test the_arena_is_a_tab_and_says_its_own_words {
- assert WIDGET_IDS[244] == "wars";
- assert WIDGET_KINDS[244] == "tab";
- assert WIDGET_PAGES[244] == "wars";
+ assert WIDGET_IDS[271] == "wars";
+ assert WIDGET_KINDS[271] == "tab";
+ assert WIDGET_PAGES[271] == "wars";
}
test the_gallery_posts_nothing {
@@ -157,8 +157,8 @@ test the_verification_shelf_is_rehearsals_and_audits {
test ideas_are_counted_apart_from_widgets {
assert IDEA_COUNT == 6;
- assert WIDGET_IDS[249] == "gatle";
- assert WIDGET_COUNT == 252;
+ assert WIDGET_IDS[276] == "gatle";
+ assert WIDGET_COUNT == 279;
}
test the_ternary_machine_shelf_runs_before_the_arena {
@@ -172,9 +172,9 @@ test every_tool_lives_under_widgets_and_keeps_its_words_on_its_page {
assert WIDGET_KINDS[9] == "tool";
assert WIDGET_PAGES[9] == "widgets/bit-diff/";
assert WIDGET_HOOKS[9] == "";
- assert WIDGET_KINDS[248] == "tool";
- assert WIDGET_CATEGORIES[248] == "game";
- assert WIDGET_PAGES[248] == "widgets/race-the-bee/";
+ assert WIDGET_KINDS[275] == "tool";
+ assert WIDGET_CATEGORIES[275] == "game";
+ assert WIDGET_PAGES[275] == "widgets/race-the-bee/";
}
test the_arithmetic_shelf_closes_the_fpga_bench {
@@ -185,3 +185,12 @@ test the_arithmetic_shelf_closes_the_fpga_bench {
assert WIDGET_CATEGORIES[124] == "fpga";
assert WIDGET_HOOKS[124] == "";
}
+
+test the_neural_network_shelf_runs_before_the_arena {
+ assert WIDGET_IDS[244] == "nn-relu";
+ assert WIDGET_KINDS[244] == "tool";
+ assert WIDGET_PAGES[244] == "widgets/nn-relu/";
+ assert WIDGET_IDS[270] == "nn-ppo";
+ assert WIDGET_CATEGORIES[270] == "compiler";
+ assert WIDGET_IDS[271] == "wars";
+}
diff --git a/apps/website/specs/widgets/nn-adam.t27 b/apps/website/specs/widgets/nn-adam.t27
new file mode 100644
index 0000000000..df20f42a52
--- /dev/null
+++ b/apps/website/specs/widgets/nn-adam.t27
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-adam.t27 -- Adam: momentum, a running scale and the bias correction
+; Source of truth for public/widgets/nn-adam/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-adam/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/adam.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 18 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets the number of steps and three gradients from -50 to 50. The path is
+; descend(n) of the spec (lr 0.1 on (x - 3)^2 from 0); the bars and tiles are first_step(g0, g1, g2,
+; which, i) on weights (1, 2, 3) at lr 0.001.
+
+; WHAT IT DOES NOT CLAIM: the bowl is one weight, as in the SGD lesson, so the two paths can be
+; compared, not ranked.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_adam;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-adam";
+pub const TITLE : str = "Adam: the optimizer most networks are trained with";
+pub const DESCRIPTION : str = "Watch Adam carry a weight from 0 to the minimum at 3, then set three gradients of any size: the bias correction makes the first step lr times their sign, whatever their size.";
+pub const IMAGE_ALT : str = "Adam card: the green path rises from 0, passes the white dashed minimum at 3, peaks near 3.18 and settles at 3.00005 by step 200. Below, three bars of the first moment m read 0.20, -4.00 and 0.05 while the weights read 0.9990, 2.0010 and 2.9990. Tiles read x 3.00005 and step 1 at 0.1000.";
+pub const HOOK : str = "Gradients 2, -40 and 0.5 all move their weights by exactly 0.001 on step 1. Adam reaches 3.00005 in 200 steps at lr 0.1.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/adam.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 18 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS_HI : u8 = 200;
+pub const K_MINIMUM : u8 = 3;
+pub const K_Y_LO : u8 = 0;
+pub const K_Y_HI : u8 = 4;
+pub const K_Y_TICKS : [5]u8 = [0, 1, 2, 3, 4];
+pub const K_STEP_TICKS : [5]u8 = [0, 50, 100, 150, 200];
+pub const K_G_LO_C : i16 = -5000;
+pub const K_G_HI_C : u16 = 5000;
+pub const K_BAR_LO : i8 = -5;
+pub const K_BAR_HI : u8 = 5;
+pub const K_BAR_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_PRESET_STEPS : [4]u8 = [200, 10, 50, 200];
+pub const K_PRESET_G0_C : [4]i16 = [200, 100, -100, 5000];
+pub const K_PRESET_G1_C : [4]i16 = [-4000, 100, -100, 1];
+pub const K_PRESET_G2_C : [4]i16 = [50, 100, -100, -5000];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["200 steps; g 2, -40, 0.5", "10 steps; equal g", "50 steps; negative g", "200 steps; g 50, 0.01, -50"];
+pub const SAY_ASK : str = "Why is Adam the optimizer most networks are trained with?";
+pub const SAY_STEPS : str = "steps";
+pub const SAY_G : str = "gradient";
+pub const SAY_W : str = "w";
+pub const SAY_PLOT_LABEL : str = "The weight after every Adam step, from 0 toward the minimum at 3";
+pub const SAY_FIRST_LABEL : str = "Step 1 on weights 1, 2, 3: the first moment m and the weight after";
+pub const SAY_PATH : str = "Adam, lr 0.1";
+pub const SAY_MIN : str = "the minimum";
+pub const SAY_X_NOW : str = "x after the steps";
+pub const SAY_X_ONE : str = "after step 1";
+pub const SAY_W_AFTER : str = "weights after step 1";
+pub const SAY_V : str = "second moment v";
+pub const SAY_NOTE : str = "m and v start at zero, so early on they are too small; dividing by 1 - beta^t corrects them. On step 1 that makes m / sqrt(v) exactly the sign of the gradient.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/adam.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_gradients_are_symmetric {
+ assert K_G_LO_C + K_G_HI_C == 0;
+ assert K_PRESET_G0_C[3] == K_G_HI_C;
+ assert K_PRESET_G2_C[3] == K_G_LO_C;
+}
+
+test the_minimum_is_inside_the_axis {
+ assert K_MINIMUM == 3;
+ assert K_Y_HI > K_MINIMUM;
+ assert K_PRESET_STEPS[0] == K_STEPS_HI;
+}
diff --git a/apps/website/specs/widgets/nn-attention.t27 b/apps/website/specs/widgets/nn-attention.t27
new file mode 100644
index 0000000000..32f3a9f038
--- /dev/null
+++ b/apps/website/specs/widgets/nn-attention.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-attention.t27 -- scaled dot-product attention with two heads and a causal mask
+; Source of truth for public/widgets/nn-attention/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-attention/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/recurrent/attention_mechanism.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 24 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader toggles the causal mask and taps a query row. Every weight is
+; forward_at(causal, 1, i) and every output forward_at(causal, 0, i) of the spec: 3 positions,
+; d_model 4, 2 heads, its own q, k and v.
+
+; WHAT IT DOES NOT CLAIM: q, k and v are the spec's fixture, not learned projections of real tokens.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_attention;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-attention";
+pub const TITLE : str = "Attention: how a word decides which other words to read";
+pub const DESCRIPTION : str = "Two heads of attention over three positions: each row is how much one query reads each key. Switch on the causal mask and a position can no longer read the future.";
+pub const IMAGE_ALT : str = "Attention card, causal: two 3 x 3 grids in blue to white. Head 0 reads 1.000 / 0.452 0.548 / 0.230 0.321 0.449 with the upper triangle at 0; head 1 1.000 / 0.434 0.566 / 0.210 0.316 0.474. Row 2 is outlined. Below, a 3 x 4 output grid; position 2 outputs 0.59 0.69 0.81 0.91.";
+pub const HOOK : str = "Causal: position 0 reads only itself (1.000), position 1 splits 0.45 / 0.55, position 2 spreads 0.23, 0.32, 0.45. Every row is a softmax.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/recurrent/attention_mechanism.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 24 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SEQ : u8 = 3;
+pub const K_D_MODEL : u8 = 4;
+pub const K_HEADS : u8 = 2;
+pub const K_PRESET_CAUSAL : [3]u8 = [1, 0, 1];
+pub const K_PRESET_Q : [3]u8 = [2, 2, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["Causal, position 2", "Full, position 2", "Causal, position 1"];
+pub const SAY_ASK : str = "How does a word decide which other words to read?";
+pub const SAY_CAUSAL : str = "causal mask";
+pub const SAY_HEAD : str = "head";
+pub const SAY_POS : str = "pos";
+pub const SAY_DIM : str = "dim";
+pub const SAY_ROWS : str = "rows are queries, columns keys";
+pub const SAY_TAP : str = "Tap a row to follow one query.";
+pub const SAY_OUT_LABEL : str = "The output: each position's weighted average of the values";
+pub const SAY_MASK : str = "mask";
+pub const SAY_MASK_ON : str = "no reading ahead";
+pub const SAY_MASK_OFF : str = "every key";
+pub const SAY_OUT_ROW : str = "output of {0}";
+pub const SAY_NOTE : str = "Each weight is softmax(q . k / sqrt(d)) over the keys of its row; the output is the values averaged by those weights. Two heads look at different halves of the vector.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/recurrent/attention_mechanism.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test two_heads_split_the_model {
+ assert K_HEADS * 2 == K_D_MODEL;
+ assert K_SEQ == 3;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_Q[0] < K_SEQ;
+ assert K_PRESET_CAUSAL[0] == 1;
+ assert K_PRESET_CAUSAL[1] == 0;
+}
diff --git a/apps/website/specs/widgets/nn-backprop.t27 b/apps/website/specs/widgets/nn-backprop.t27
new file mode 100644
index 0000000000..9b1fde1101
--- /dev/null
+++ b/apps/website/specs/widgets/nn-backprop.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-backprop.t27 -- backpropagation through one dense layer
+; Source of truth for public/widgets/nn-backprop/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-backprop/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/dense_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 19 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets the output gradients g0 and g1 from -2 to 2. Edges are
+; backward_at(g0, g1, 1, o * 3 + i), input nodes backward_at(g0, g1, 0, i), bias tiles
+; backward_at(g0, g1, 2, o), and the loss loss_at = g . y at x and at x with x0 moved by 1.
+
+; WHAT IT DOES NOT CLAIM: the layer is the spec's fixture (W = [[1, 2, 3], [-1, 0, 1]], b = (0.5,
+; -0.5)), linear; the inputs are repeated as K_X.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_backprop;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-backprop";
+pub const TITLE : str = "Backpropagation: how blame flows back through a layer";
+pub const DESCRIPTION : str = "Set how much the loss wants each of two outputs to move: every weight gets that times its input, every input gets W transposed times it. The edges and nodes show the numbers.";
+pub const IMAGE_ALT : str = "Backprop card: a dense layer of 3 inputs and 2 outputs with g = (1, 0.5). Six edges are coloured by their weight gradients 1, 2, 3 and 0.5, 1, 1.5; the input nodes hold 0.5, 2.0 and 3.5. Tiles read input gradients 0.50 2.00 3.50, bias gradients 1.00 0.50, loss 15.250 and with x0 moved by 1 15.750.";
+pub const HOOK : str = "With g = (1, 0.5) on inputs (1, 2, 3), the weights get 1, 2, 3 and 0.5, 1, 1.5, and the inputs get 0.5, 2, 3.5. Two products are the whole rule.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/dense_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 19 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X : [3]u8 = [1, 2, 3];
+pub const K_X_NUDGED : [3]u8 = [2, 2, 3];
+pub const K_G_LO_C : i16 = -200;
+pub const K_G_HI_C : u16 = 200;
+pub const K_PRESET_G0_C : [4]i16 = [100, 100, 0, -200];
+pub const K_PRESET_G1_C : [4]i16 = [50, 0, 100, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["g = (1, 0.5)", "Only output 0", "Only output 1", "g = (-2, 1)"];
+pub const SAY_ASK : str = "How does blame for an error flow back through a layer?";
+pub const SAY_G : str = "g";
+pub const SAY_NET_LABEL : str = "Three inputs, two outputs; edges coloured by their weight gradient, inputs by theirs";
+pub const SAY_NET_CAP : str = "Edge label: g_o x_i. Input node: sum over o of W[o][i] g_o. Pink is negative, green positive.";
+pub const SAY_GRAD_IN : str = "input gradients";
+pub const SAY_GRAD_B : str = "bias gradients";
+pub const SAY_LOSS : str = "loss g . y";
+pub const SAY_LOSS_NUDGED : str = "with x0 + 1";
+pub const SAY_NOTE : str = "The layer is linear, so moving x0 by 1 moves the loss by exactly its input gradient: compare the last two tiles. Chain this rule layer by layer and you have backpropagation.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/dense_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_inputs_are_the_spec_fixture {
+ assert K_X[0] == 1;
+ assert K_X[2] == 3;
+ assert K_X_NUDGED[0] == K_X[0] + 1;
+ assert K_X_NUDGED[2] == K_X[2];
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_G0_C[3] == K_G_LO_C;
+ assert K_PRESET_G1_C[2] <= K_G_HI_C;
+ assert K_G_LO_C + K_G_HI_C == 0;
+}
diff --git a/apps/website/specs/widgets/nn-batchnorm.t27 b/apps/website/specs/widgets/nn-batchnorm.t27
new file mode 100644
index 0000000000..5c886e03c8
--- /dev/null
+++ b/apps/website/specs/widgets/nn-batchnorm.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-batchnorm.t27 -- batch normalisation: features on different scales, made comparable
+; Source of truth for public/widgets/nn-batchnorm/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-batchnorm/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/batchnorm_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 12 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader reads the batch before and after and sets gamma and the running
+; statistics. The normalised batch is train_at(b, f) and the inference bars forward_at(gamma,
+; shifted, i) on inputs (1, 2, 3), beta 0.25.
+
+; WHAT IT DOES NOT CLAIM: the batch on the left is the spec's fixture, repeated as K_BATCH; the
+; normalised values come from the compiled fn.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_batchnorm;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-batchnorm";
+pub const TITLE : str = "Batch norm: features on very different scales, made comparable";
+pub const DESCRIPTION : str = "A batch of 4 samples whose 3 features live on scales 1, 10 and 1: batch norm gives each feature mean 0 and variance 1. Then set gamma and the running statistics for inference.";
+pub const IMAGE_ALT : str = "Batch-norm card: on the left the spec's batch, 4 samples by 3 features (1, 2, 3, 4 / 10, 20, 30, 40 / -1, -1, 1, 1); on the right the normalised batch, the first two columns both -1.34, -0.45, 0.45, 1.34 and the third -1.00, -1.00, 1.00, 1.00. Below, gamma 1 at inference gives 1.25, 2.25, 3.25.";
+pub const HOOK : str = "Features 1..4 and 10..40 come out identical, -1.34, -0.45, 0.45, 1.34: batch norm keeps the shape of each feature and throws away its scale.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/batchnorm_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 12 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_BATCH_ROWS : u8 = 4;
+pub const K_FEATURES : u8 = 3;
+pub const K_BATCH : [12]i8 = [1, 10, -1, 2, 20, -1, 3, 30, 1, 4, 40, 1];
+pub const K_GAMMA_HI_C : u16 = 300;
+pub const K_INFER_INPUT : [3]u8 = [1, 2, 3];
+pub const K_BAR_LO_C : i16 = -200;
+pub const K_BAR_HI_C : u16 = 1000;
+pub const K_BAR_TICKS : [5]i8 = [-2, 0, 2, 5, 10];
+pub const K_PRESET_GAMMA_C : [4]u16 = [100, 200, 100, 50];
+pub const K_PRESET_SHIFTED : [4]u8 = [0, 0, 1, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["gamma 1, mean 0 / var 1", "gamma 2", "gamma 1, mean 2 / var 4", "gamma 0.5, mean 2 / var 4"];
+pub const SAY_ASK : str = "What happens when one feature is ten times larger than another?";
+pub const SAY_GAMMA : str = "gamma";
+pub const SAY_SHIFTED : str = "running mean 2, variance 4";
+pub const SAY_BEFORE_LABEL : str = "The batch: 4 samples x 3 features";
+pub const SAY_AFTER_LABEL : str = "After batch norm";
+pub const SAY_INFER_LABEL : str = "At inference: inputs 1, 2, 3 through the running statistics";
+pub const SAY_FEATURE : str = "feature";
+pub const SAY_SAMPLE : str = "sample";
+pub const SAY_INPUT : str = "input";
+pub const SAY_COL : str = "feature {0} after";
+pub const SAY_INFER : str = "inference out";
+pub const SAY_NOTE : str = "Training normalises with the batch's own mean and variance; inference cannot, so it uses running averages kept during training. gamma and beta are learned, so the layer can undo the normalisation where that helps.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/batchnorm_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_batch_is_the_spec_fixture {
+ assert K_BATCH_ROWS * K_FEATURES == 12;
+ assert K_BATCH[1] == 10;
+ assert K_BATCH[10] == 40;
+ assert K_BATCH[2] + 1 == 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_GAMMA_C[1] <= K_GAMMA_HI_C;
+ assert K_PRESET_SHIFTED[2] == 1;
+ assert K_INFER_INPUT[2] == 3;
+}
diff --git a/apps/website/specs/widgets/nn-bce.t27 b/apps/website/specs/widgets/nn-bce.t27
new file mode 100644
index 0000000000..c1f2b414ae
--- /dev/null
+++ b/apps/website/specs/widgets/nn-bce.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-bce.t27 -- binary cross-entropy: why a confident wrong answer costs so much
+; Source of truth for public/widgets/nn-bce/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-bce/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/loss/binary_crossentropy_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 14 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader moves three probabilities from 0.01 to 0.99. The loss is loss_of(a, b, c,
+; false) against the spec's targets (1, 0, 1), the bars grad_of, and the curve loss_of over the
+; first prediction; one tile feeds logits of 0 through the spec's sigmoid.
+
+; WHAT IT DOES NOT CLAIM: the spec clips each probability to [1e-7, 1 - 1e-7], so the curve is
+; finite at the ends of this window.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_bce;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-bce";
+pub const TITLE : str = "Cross-entropy: why a confident wrong answer costs so much";
+pub const DESCRIPTION : str = "Set three predicted probabilities against the answers 1, 0, 1: the loss is -log of the chance given to the truth. Drag toward a confident mistake and the cost climbs without bound.";
+pub const IMAGE_ALT : str = "Cross-entropy card: predictions 0.8, 0.3 and 0.9 against targets 1, 0, 1. A red curve shows the loss as prediction 0 runs from 0.01 to 0.99, rising steeply on the left, with the cursor at 0.8; gradient bars read -1.25, 1.43 and -1.11. Tiles read loss 0.2284, sure and right 0.0101, sure and wrong 4.6052, logits at 0 giving 0.6931.";
+pub const HOOK : str = "Predicting 0.99 where the answer is 1 costs 0.01; predicting 0.01 costs 4.6. The cost of a confident mistake is what trains a classifier.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/loss/binary_crossentropy_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 14 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_TARGETS : [3]u8 = [1, 0, 1];
+pub const K_P_LO_C : u8 = 1;
+pub const K_P_HI_C : u8 = 99;
+pub const K_X_TICKS_C : [5]u8 = [1, 25, 50, 75, 99];
+pub const K_LOSS_HI_C : u16 = 200;
+pub const K_LOSS_TICKS_C : [5]u8 = [0, 50, 100, 150, 200];
+pub const K_GRAD_LO_C : i16 = -1000;
+pub const K_GRAD_HI_C : u16 = 1000;
+pub const K_GRAD_TICKS : [5]i8 = [-10, -5, 0, 5, 10];
+pub const K_PRESET_A_C : [4]u8 = [80, 99, 10, 50];
+pub const K_PRESET_B_C : [4]u8 = [30, 1, 90, 50];
+pub const K_PRESET_C_C : [4]u8 = [90, 99, 10, 50];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["0.8, 0.3, 0.9", "Sure and right", "Sure and wrong", "Undecided"];
+pub const SAY_ASK : str = "Why does a confident wrong answer cost so much?";
+pub const SAY_PRED : str = "p";
+pub const SAY_TARGET : str = "answer";
+pub const SAY_CURVE_LABEL : str = "The loss as the first prediction moves, the others held";
+pub const SAY_GRAD_LABEL : str = "The gradient for each prediction";
+pub const SAY_LOSS : str = "loss";
+pub const SAY_SURE_RIGHT : str = "0.99 where right";
+pub const SAY_SURE_WRONG : str = "0.01 where right";
+pub const SAY_LOGITS : str = "logits 0, 0, 0";
+pub const SAY_NOTE : str = "The loss of one case is -(t log p + (1 - t) log(1 - p)). The gradient of a prediction is huge where the model is sure and wrong, and small where it is sure and right.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/loss/binary_crossentropy_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_targets_are_the_spec_fixture {
+ assert K_TARGETS[0] == 1;
+ assert K_TARGETS[1] == 0;
+ assert K_TARGETS[2] == 1;
+}
+
+test the_window_stays_inside_zero_and_one {
+ assert K_P_LO_C > 0;
+ assert K_P_HI_C < 100;
+ assert K_PRESET_B_C[1] == K_P_LO_C;
+ assert K_PRESET_A_C[1] == K_P_HI_C;
+}
diff --git a/apps/website/specs/widgets/nn-conv.t27 b/apps/website/specs/widgets/nn-conv.t27
new file mode 100644
index 0000000000..df845ff1df
--- /dev/null
+++ b/apps/website/specs/widgets/nn-conv.t27
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-conv.t27 -- a window over a picture: 2 x 2 average pooling
+; Source of truth for public/widgets/nn-conv/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-conv/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/avgpool2d_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 10 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader taps a window, on the image or on the output, and toggles the gradient
+; view. Outputs are pool4_at(i), the output side calc_output_size(4) and the gradient
+; backward4_at(i) for the spec's output gradients (4, 8, 12, 16).
+
+; WHAT IT DOES NOT CLAIM: this is AVERAGE pooling with a fixed 2 x 2 window, not a convolution with
+; a learned kernel: the sliding conv2d spec is gHashTag/t27#10612, not merged when this page was
+; made.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_conv;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-conv";
+pub const TITLE : str = "Patches of a picture: a 2 x 2 window, averaged";
+pub const DESCRIPTION : str = "Tap a window on the 4 x 4 image: each output is the average of its 2 x 2 patch, two cells apart. Then switch to the gradient and see each output's blame shared out over its patch.";
+pub const IMAGE_ALT : str = "Window card: a 4 x 4 image holding 0 to 15, with the 2 x 2 patch 2, 3, 6, 7 outlined and the rest dimmed; beside it the 2 x 2 output 2.50, 4.50, 10.50, 12.50 with 4.50 outlined. Tiles read window 2 / 4, average 4.50, output 2 x 2 and gradient share 2.00.";
+pub const HOOK : str = "Pictures are read in patches. Here a 2 x 2 window steps by 2 over a 4 x 4 image and averages: 2.5, 4.5, 10.5, 12.5. A learned kernel comes next.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/avgpool2d_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 10 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SIDE : u8 = 4;
+pub const K_KERNEL : u8 = 2;
+pub const K_DEFAULT_KERNEL : u8 = 2;
+pub const K_DEFAULT_STRIDE : u8 = 2;
+pub const K_WIN_ROW : [4]u8 = [0, 0, 2, 2];
+pub const K_WIN_COL : [4]u8 = [0, 2, 0, 2];
+pub const K_PRESET_WIN : [3]u8 = [1, 0, 3];
+pub const K_PRESET_GRAD : [3]u8 = [0, 0, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["Window 2", "Window 1", "The gradient of window 4"];
+pub const SAY_ASK : str = "How does a network look at a picture: all at once, or in patches?";
+pub const SAY_SHOW_GRAD : str = "show the gradient";
+pub const SAY_IN_LABEL : str = "The 4 x 4 input: tap a patch";
+pub const SAY_GRAD_LABEL : str = "The gradient each input receives";
+pub const SAY_OUT_LABEL : str = "The 2 x 2 output: one number per patch";
+pub const SAY_WINDOW : str = "window";
+pub const SAY_AVERAGE : str = "its average";
+pub const SAY_OUT_SIDE : str = "output";
+pub const SAY_GRAD_SHARE : str = "gradient per input";
+pub const SAY_NOTE : str = "Average pooling, not yet a convolution: every window weighs its four cells equally. A convolution would weigh them with a learned 2 x 2 kernel and usually step by 1; that spec (t27#10612) is still open.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/avgpool2d_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_windows_tile_the_image {
+ assert K_SIDE == 4;
+ assert K_KERNEL == 2;
+ assert K_WIN_ROW[3] + K_KERNEL == K_SIDE;
+ assert K_WIN_COL[1] + K_KERNEL == K_SIDE;
+}
+
+test the_window_steps_by_its_own_size {
+ assert K_DEFAULT_KERNEL == K_DEFAULT_STRIDE;
+ assert K_WIN_COL[1] == K_DEFAULT_STRIDE;
+ assert K_PRESET_WIN[2] == 3;
+}
diff --git a/apps/website/specs/widgets/nn-dropout.t27 b/apps/website/specs/widgets/nn-dropout.t27
new file mode 100644
index 0000000000..dcee5951e5
--- /dev/null
+++ b/apps/website/specs/widgets/nn-dropout.t27
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-dropout.t27 -- dropout, inverted and classic: training with neurons switched off
+; Source of truth for public/widgets/nn-dropout/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-dropout/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/dropout_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 9 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader toggles training and inverted scaling. The rows are forward_at(false,
+; true, i) (the identity, so the input), forward_at(training, scaled, i) and, while training,
+; backward_at(scaled, i).
+
+; WHAT IT DOES NOT CLAIM: the mask is the spec's fixed one (keep, drop, keep, drop) with p 0.5, not
+; a random draw; the point is the scaling, not the sampling.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_dropout;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-dropout";
+pub const TITLE : str = "Dropout: training a network with neurons switched off";
+pub const DESCRIPTION : str = "Switch between training and inference, inverted and classic dropout: units 1 and 3 are dropped, the survivors are scaled so that the layer's output keeps its expected size.";
+pub const IMAGE_ALT : str = "Dropout card, training with inverted dropout: three rows of four cells. Input 1, 2, 3, 4; output 2, 0, 6, 0 with units 1 and 3 dimmed; gradient 2, 0, 2, 0. Tiles read mode training, a kept unit 2.00, a dropped unit 0.00 and the kept gradient 2.00.";
+pub const HOOK : str = "With p = 0.5, inverted dropout doubles the units it keeps while training (1, 2, 3, 4 becomes 2, 0, 6, 0) and leaves inference alone.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/dropout_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 9 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_UNITS : u8 = 4;
+pub const K_PRESET_TRAIN : [4]u8 = [1, 1, 0, 0];
+pub const K_PRESET_SCALED : [4]u8 = [1, 0, 1, 0];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Training, inverted", "Training, classic", "Inference, inverted", "Inference, classic"];
+pub const SAY_ASK : str = "Why would a network train with some of its neurons switched off?";
+pub const SAY_TRAINING : str = "training";
+pub const SAY_SCALED : str = "inverted (scale while training)";
+pub const SAY_GRID_LABEL : str = "Input, output and gradient of four units through dropout";
+pub const SAY_IN : str = "input";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "gradient";
+pub const SAY_UNIT : str = "unit";
+pub const SAY_MODE : str = "mode";
+pub const SAY_MODE_TRAIN : str = "training";
+pub const SAY_MODE_INFER : str = "inference";
+pub const SAY_KEPT_UNIT : str = "unit 0, kept";
+pub const SAY_DROPPED_UNIT : str = "unit 1, dropped";
+pub const SAY_GRAD_KEPT : str = "gradient, kept unit";
+pub const SAY_NOTE : str = "Dropping random units forces the rest not to lean on any one of them. Inverted dropout scales by 1 / (1 - p) while training so that inference is a plain pass; classic dropout scales by 1 - p at inference instead.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/dropout_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test four_units_four_modes {
+ assert K_UNITS == 4;
+ assert K_PRESET_TRAIN[0] == 1;
+ assert K_PRESET_TRAIN[3] == 0;
+}
+
+test every_mode_is_offered {
+ assert K_PRESET_SCALED[0] != K_PRESET_SCALED[1];
+ assert K_PRESET_SCALED[2] != K_PRESET_SCALED[3];
+ assert K_START_PRESET == 0;
+}
diff --git a/apps/website/specs/widgets/nn-elu.t27 b/apps/website/specs/widgets/nn-elu.t27
new file mode 100644
index 0000000000..9d38c009ac
--- /dev/null
+++ b/apps/website/specs/widgets/nn-elu.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-elu.t27 -- ELU: a negative side with a floor at -alpha
+; Source of truth for public/widgets/nn-elu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-elu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/elu_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 6 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -5 to 3 and sets alpha from 0.1 to 2. The curve is
+; forward(x, alpha), the slope derivative(x, alpha), and the floor line is forward far to the left
+; (x = -100).
+
+; WHAT IT DOES NOT CLAIM: the floor is read as forward(-100, alpha), which is -alpha to f32
+; precision, not the limit itself.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_elu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-elu";
+pub const TITLE : str = "ELU: a negative side with a floor";
+pub const DESCRIPTION : str = "Drag along ELU: x above zero, alpha (e^x - 1) below. The negative side never falls under -alpha, so outputs average closer to zero, and its slope never quite dies.";
+pub const IMAGE_ALT : str = "ELU card at alpha 1, x = -3: the green curve runs along the diagonal for positive x and bends down toward the red dashed floor at -1 on the left; the dashed gold slope is 1 on the right and decays toward 0 on the left. Tiles read -0.9502, slope 0.0498, floor -1.0000 and the spec batch (-1, 0, 2) as -0.632 0.000 2.000.";
+pub const HOOK : str = "ELU(-3) with alpha 1 is -0.9502, already 95% of the way to its floor of -1, and its slope there is still 0.0498, not 0.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/elu_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 6 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -500;
+pub const K_X_HI_C : i16 = 300;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_Y_LO_C : i16 = -220;
+pub const K_Y_HI_C : i16 = 300;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 3];
+pub const K_Y_TICKS : [6]i8 = [-2, -1, 0, 1, 2, 3];
+pub const K_ALPHA_LO_C : u8 = 10;
+pub const K_ALPHA_HI_C : u8 = 200;
+pub const K_ALPHA_STEP_C : u8 = 5;
+pub const K_FAR_LEFT : u8 = 100;
+pub const K_PRESET_X_C : [4]i16 = [-300, -300, 150, -100];
+pub const K_PRESET_ALPHA_C : [4]u8 = [100, 50, 100, 200];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["alpha 1, x = -3", "alpha 0.5, x = -3", "alpha 1, x = 1.5", "alpha 2, x = -1"];
+pub const SAY_ASK : str = "What if the negative side had a floor instead of a cliff?";
+pub const SAY_X : str = "input x";
+pub const SAY_ALPHA : str = "alpha";
+pub const SAY_F : str = "ELU(x)";
+pub const SAY_DF : str = "slope";
+pub const SAY_FLOOR : str = "floor";
+pub const SAY_PLOT_LABEL : str = "ELU of x, its slope and its floor from -5 to 3";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_BATCH : str = "spec batch -1 0 2";
+pub const SAY_NOTE : str = "ReLU throws the negative side away; ELU keeps a soft version of it, so a unit stuck below zero still moves. alpha is how deep the floor lies.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/elu_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_floor_is_far_left {
+ assert K_FAR_LEFT == 100;
+ assert K_FAR_LEFT * 100 + K_X_LO_C > 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_ALPHA_C[3] == K_ALPHA_HI_C;
+ assert K_PRESET_ALPHA_C[1] >= K_ALPHA_LO_C;
+ assert K_Y_LO_C + 200 < 0;
+}
diff --git a/apps/website/specs/widgets/nn-embedding.t27 b/apps/website/specs/widgets/nn-embedding.t27
new file mode 100644
index 0000000000..309aaa6fd1
--- /dev/null
+++ b/apps/website/specs/widgets/nn-embedding.t27
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-embedding.t27 -- the embedding table: a word becomes a row of numbers
+; Source of truth for public/widgets/nn-embedding/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-embedding/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/embedding_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 8 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader picks two tokens from four. The table is lookup_at(v, v, j) for every row
+; v, the output lookup_at(a, b, i), and the gradient table grad_at(a, b, i) for the spec's gradient
+; rows (1, 2) and (3, 4).
+
+; WHAT IT DOES NOT CLAIM: the table is the spec's 4 x 2 fixture (row v = 10 v, 10 v + 1); a real one
+; has a row per word and hundreds of columns.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_embedding;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-embedding";
+pub const TITLE : str = "Embeddings: how a word becomes a row of numbers";
+pub const DESCRIPTION : str = "Pick two tokens: each one copies its row out of the table. Then the gradient flows back only into the rows that were read, and a token read twice collects both.";
+pub const IMAGE_ALT : str = "Embedding card: token 2 read twice from a 4 x 2 table whose rows are (0, 1), (10, 11), (20, 21), (30, 31). Row 2 is outlined and the others dimmed; the output is (20, 21) twice, and the gradient table is all zero except row 2, which holds 4 and 6.";
+pub const HOOK : str = "A token is an index into a table, nothing more. Read token 2 twice and its row's gradient is (1 + 3, 2 + 4) = (4, 6); the other rows get 0.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/embedding_layer.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 8 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_VOCAB : u8 = 4;
+pub const K_WIDTH : u8 = 2;
+pub const K_PRESET_A : [3]u8 = [2, 1, 0];
+pub const K_PRESET_B : [3]u8 = [2, 3, 2];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["The same token twice", "Tokens 1 and 3", "Tokens 0 and 2"];
+pub const SAY_ASK : str = "How does a word become numbers a network can use?";
+pub const SAY_TOKEN_NAMES : [4]str = ["token 0", "token 1", "token 2", "token 3"];
+pub const SAY_FIRST : str = "first token";
+pub const SAY_SECOND : str = "second token";
+pub const SAY_DIM : str = "dim";
+pub const SAY_TABLE_LABEL : str = "The embedding table: one row per token";
+pub const SAY_OUT_LABEL : str = "What the layer outputs";
+pub const SAY_GRAD_LABEL : str = "Where the gradient lands";
+pub const SAY_ROWS_TOUCHED : str = "rows read";
+pub const SAY_ONE_ROW : str = "one row, twice";
+pub const SAY_TWO_ROWS : str = "two rows";
+pub const SAY_VECTORS : str = "output";
+pub const SAY_GRAD_ROW : str = "gradient of {0}";
+pub const SAY_NOTE : str = "An embedding layer has no multiply at all: it copies rows. Learning changes only the rows it copied, which is why a rare word learns slowly.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/embedding_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_table_is_the_spec_fixture {
+ assert K_VOCAB == 4;
+ assert K_WIDTH == 2;
+ assert K_VOCAB * K_WIDTH == 8;
+}
+
+test the_presets_name_real_tokens {
+ assert K_PRESET_A[0] == K_PRESET_B[0];
+ assert K_PRESET_B[1] < K_VOCAB;
+ assert K_PRESET_A[1] != K_PRESET_B[1];
+}
diff --git a/apps/website/specs/widgets/nn-encoder.t27 b/apps/website/specs/widgets/nn-encoder.t27
new file mode 100644
index 0000000000..66df55b1cc
--- /dev/null
+++ b/apps/website/specs/widgets/nn-encoder.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-encoder.t27 -- a transformer encoder block, post-norm and pre-norm
+; Source of truth for public/widgets/nn-encoder/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-encoder/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/transformer/encoder_block.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 26 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader toggles post-norm and pre-norm. The output grid is block_at(pre, 0, i),
+; the row means and variances block_at(pre, 1 or 2, row), and the attention grids attention_at(1,
+; i): 3 positions, d_model 4, 2 heads, d_ff 8, the spec's weights.
+
+; WHAT IT DOES NOT CLAIM: one block with the spec's fixed weights, not a trained model; a
+; transformer stacks dozens.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_encoder;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-encoder";
+pub const TITLE : str = "An encoder block: what one transformer layer does";
+pub const DESCRIPTION : str = "Three positions through one transformer block: attention, a residual, a feed-forward net and layer norm. Post-norm ends every row at mean 0 and variance 1; pre-norm does not.";
+pub const IMAGE_ALT : str = "Encoder card, post-norm: a 3 x 4 output grid in pink and green, every row centred on 0. Tiles read row means 0.000 0.000 0.000 and variances 1.000 1.000 1.000. Below, two 3 x 3 attention grids of the block, head 0 row 0 reading 0.401 0.198 0.401.";
+pub const HOOK : str = "Post-norm rows come out at mean 0 and variance 1.000; pre-norm rows at variances 2.73, 3.33 and 0.96, because its last step is a residual add.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/transformer/encoder_block.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 26 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SEQ : u8 = 3;
+pub const K_D_MODEL : u8 = 4;
+pub const K_HEADS : u8 = 2;
+pub const K_PRESET_PRE : [2]u8 = [0, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [2]str = ["Post-norm", "Pre-norm"];
+pub const SAY_ASK : str = "What does one layer of a transformer actually do to its input?";
+pub const SAY_PRE_NORM : str = "pre-norm";
+pub const SAY_HEAD : str = "head";
+pub const SAY_POS : str = "pos";
+pub const SAY_DIM : str = "dim";
+pub const SAY_OUT_LABEL : str = "The block's output: 3 positions x 4 dimensions";
+pub const SAY_ATT_LABEL : str = "The block's self-attention weights";
+pub const SAY_MEANS : str = "row means";
+pub const SAY_VARS : str = "row variances";
+pub const SAY_ORDER : str = "order";
+pub const SAY_ORDER_POST : str = "attend, add, norm, FFN, add, norm";
+pub const SAY_ORDER_PRE : str = "norm, attend, add, norm, FFN, add";
+pub const SAY_ATT_ROW : str = "head 0, pos 0 reads";
+pub const SAY_NOTE : str = "The residual adds keep the input flowing through, so a deep stack can train; the norms keep the numbers in range. Where the norm sits decides what comes out.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/transformer/encoder_block.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test two_heads_split_the_model {
+ assert K_HEADS * 2 == K_D_MODEL;
+ assert K_SEQ == 3;
+}
+
+test both_orders_are_offered {
+ assert K_PRESET_PRE[0] == 0;
+ assert K_PRESET_PRE[1] == 1;
+}
diff --git a/apps/website/specs/widgets/nn-gelu.t27 b/apps/website/specs/widgets/nn-gelu.t27
new file mode 100644
index 0000000000..7a202a415b
--- /dev/null
+++ b/apps/website/specs/widgets/nn-gelu.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-gelu.t27 -- GELU, exact and tanh form: the bend transformers use
+; Source of truth for public/widgets/nn-gelu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-gelu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/gelu_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 4 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -4 to 3. The exact curve is forward_at(x, false) through the
+; spec's erf, the dashed curve forward_at(x, true), the tanh form, and the dotted curve backward_at,
+; the slope.
+
+; WHAT IT DOES NOT CLAIM: the erf of the spec is the Abramowitz-Stegun 7.1.26 fit, good to 1.5e-7,
+; so the exact curve is exact to that.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_gelu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-gelu";
+pub const TITLE : str = "GELU: why transformers bend a little below zero";
+pub const DESCRIPTION : str = "Drag along GELU: x weighed by the chance a normal draw is below it. It dips to about -0.17 before it rises, and the tanh shortcut stays within a few thousandths of the exact form.";
+pub const IMAGE_ALT : str = "GELU card at x = -0.75: the thick green exact curve and the dashed pink tanh form lie on top of each other from -4 to 3, dipping just under zero near -0.75; the dotted cyan slope crosses zero there. Tiles read exact -0.16997, tanh form -0.17004 and slopes 0.0008.";
+pub const HOOK : str = "GELU(-0.75) = -0.16997 exactly and -0.17004 by the tanh form. A smooth ReLU with a small dip, used in BERT and GPT.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/gelu_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 4 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -400;
+pub const K_X_HI_C : i16 = 300;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_Y_LO_C : i16 = -50;
+pub const K_Y_HI_C : i16 = 300;
+pub const K_X_TICKS : [8]i8 = [-4, -3, -2, -1, 0, 1, 2, 3];
+pub const K_Y_TICKS_C : [4]i16 = [0, 100, 200, 300];
+pub const K_PRESET_X_C : [4]i16 = [-75, 150, -300, 0];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["The dip, x = -0.75", "x = 1.5", "Far left, x = -3", "x = 0"];
+pub const SAY_ASK : str = "Why do transformers use a curve that bends a little below zero?";
+pub const SAY_X : str = "input x";
+pub const SAY_EXACT : str = "GELU, exact";
+pub const SAY_TANH : str = "GELU, tanh form";
+pub const SAY_DF : str = "slope";
+pub const SAY_DF_TANH : str = "slope, tanh form";
+pub const SAY_PLOT_LABEL : str = "GELU of x, exact and tanh form, and its slope from -4 to 3";
+pub const SAY_NOTE : str = "GELU(x) = x Phi(x): a positive input mostly passes, a large negative one mostly vanishes, and in between the curve dips. Unlike ReLU its slope is smooth everywhere.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/gelu_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_dip_is_in_range {
+ assert K_PRESET_X_C[0] < 0;
+ assert K_PRESET_X_C[0] > K_X_LO_C;
+ assert K_Y_LO_C < 0;
+}
+
+test the_window_is_wider_on_the_left {
+ assert K_X_LO_C + 400 == 0;
+ assert K_X_HI_C == 300;
+ assert K_X_TICKS[0] * 100 == K_X_LO_C;
+}
diff --git a/apps/website/specs/widgets/nn-huber.t27 b/apps/website/specs/widgets/nn-huber.t27
new file mode 100644
index 0000000000..2db4401c30
--- /dev/null
+++ b/apps/website/specs/widgets/nn-huber.t27
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-huber.t27 -- the Huber loss: square inside delta, linear outside
+; Source of truth for public/widgets/nn-huber/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-huber/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/loss/huber_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 15 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the error from -4 to 4 and sets delta from 0.25 to 3. The green
+; curve is huber_one(e, delta), the dashed curve huber_one(e, 1000), which never leaves its
+; quadratic part on this window, and the tiles huber_two on the spec's pair.
+
+; WHAT IT DOES NOT CLAIM: the 30-error figure in the hook is the same two fns at e = 30, read off
+; the compiled logic, not a measurement of a model.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_huber;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-huber";
+pub const TITLE : str = "Huber loss: a loss that does not panic at an outlier";
+pub const DESCRIPTION : str = "Drag the error along: Huber is half its square inside delta and only grows linearly outside, so one wild miss pulls on the model with a bounded force. The dashed curve is the pure square.";
+pub const IMAGE_ALT : str = "Huber card at delta 1, error 3: the green Huber curve is a small parabola between the cyan dashed lines at -1 and 1 and then two straight arms; the dashed gold square climbs much faster. Tiles read Huber 2.5000, square 4.5000 and, for the spec's pair of errors (0.5, 3), 1.3125 against 2.3125.";
+pub const HOOK : str = "An error of 3 costs 2.5 under Huber with delta 1 and 4.5 squared. At 30 the gap is 29.5 against 450: why robust regression uses it.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/loss/huber_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 15 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_E_LO_C : i16 = -400;
+pub const K_E_HI_C : u16 = 400;
+pub const K_Y_HI_C : u16 = 800;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [5]u8 = [0, 2, 4, 6, 8];
+pub const K_DELTA_LO_C : u8 = 25;
+pub const K_DELTA_HI_C : u16 = 300;
+pub const K_WIDE : u16 = 1000;
+pub const K_PAIR_C : [2]u16 = [50, 300];
+pub const K_PRESET_E_C : [4]i16 = [300, 50, -300, 200];
+pub const K_PRESET_DELTA_C : [4]u16 = [100, 100, 50, 300];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Error 3, delta 1", "Error 0.5, delta 1", "Error -3, delta 0.5", "Error 2, delta 3"];
+pub const SAY_ASK : str = "Can a loss stay calm when one data point is far off?";
+pub const SAY_E : str = "error e";
+pub const SAY_DELTA : str = "delta";
+pub const SAY_HUBER : str = "Huber";
+pub const SAY_SQUARE : str = "half the square";
+pub const SAY_PLOT_LABEL : str = "Huber loss and half the squared error as the error goes from -4 to 4";
+pub const SAY_PAIR : str = "Huber of ({0}, {1})";
+pub const SAY_PAIR_SQ : str = "square of ({0}, {1})";
+pub const SAY_NOTE : str = "Inside delta Huber is the square; outside it is delta |e| - delta^2 / 2, joined so that value and slope agree. Its gradient never grows past delta.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/loss/huber_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_wide_delta_never_clips_the_window {
+ assert K_WIDE * 100 > K_E_HI_C;
+ assert K_DELTA_HI_C < K_WIDE * 100;
+}
+
+test the_pair_is_the_spec_example {
+ assert K_PAIR_C[0] == 50;
+ assert K_PAIR_C[1] == 300;
+ assert K_PRESET_DELTA_C[3] == K_DELTA_HI_C;
+}
diff --git a/apps/website/specs/widgets/nn-lamb.t27 b/apps/website/specs/widgets/nn-lamb.t27
new file mode 100644
index 0000000000..4d550fb9e8
--- /dev/null
+++ b/apps/website/specs/widgets/nn-lamb.t27
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-lamb.t27 -- LAMB: a layer-wise trust ratio for large batches
+; Source of truth for public/widgets/nn-lamb/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-lamb/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/lamb.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 21 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the scale of the weights (1, 2) from 0.1 to 10. Every point is
+; step_norm(scale), the norm of the LAMB update for gradients (0.1, 0.2) at lr 0.001, and the tiles
+; two_layers, one step on two layers at once.
+
+; WHAT IT DOES NOT CLAIM: the clip at a norm of 10 is the spec's DEFAULT_CLIP_THRESHOLD; the flat
+; part is that cap, not a property of every LAMB.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_lamb;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-lamb";
+pub const TITLE : str = "LAMB: one learning rate for layers of every size";
+pub const DESCRIPTION : str = "Scale a layer's weights and watch its LAMB step: it grows with the weight norm, so every layer moves by the same fraction of itself, until the norm passes 10 and the step stops growing.";
+pub const IMAGE_ALT : str = "LAMB card at scale 2: a green line rises straight from 0 at scale 0 and bends flat at 0.010 once the scale passes about 4.5, with the cursor at 0.00447. Tiles read step norm 0.004472, ten times the weights 0.010000, and the spec's two layers after one step 0.9984 1.9984 and 2.9965 3.9964.";
+pub const HOOK : str = "Weights (2, 4) take a step of 0.00447 and weights (6, 12) one of 0.01: a step is 0.001 of the layer's norm, capped at 10. That is how LAMB trains large batches.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/lamb.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 21 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SCALE_LO_C : u8 = 10;
+pub const K_SCALE_HI_C : u16 = 1000;
+pub const K_Y_HI_U : u8 = 12;
+pub const K_Y_TICKS_U : [5]u8 = [0, 3, 6, 9, 12];
+pub const K_X_TICKS : [6]u8 = [0, 2, 4, 6, 8, 10];
+pub const K_PRESET_SCALE_C : [4]u16 = [200, 600, 10, 1000];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["scale 2", "scale 6, capped", "scale 0.1", "scale 10"];
+pub const SAY_ASK : str = "Can one learning rate suit a tiny layer and a huge one?";
+pub const SAY_SCALE : str = "weight scale";
+pub const SAY_NORM : str = "step norm";
+pub const SAY_PLOT_LABEL : str = "The norm of one LAMB step as the layer's weights are scaled";
+pub const SAY_NORM_X10 : str = "weights x 10";
+pub const SAY_LAYER_A : str = "layer 1 after a step";
+pub const SAY_LAYER_B : str = "layer 2 after a step";
+pub const SAY_NOTE : str = "LAMB computes an Adam-style update for each layer, then rescales it by the trust ratio min(||w||, 10) / ||update||. A big layer takes a big step and a small layer a small one.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/lamb.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_window_reaches_the_cap {
+ assert K_SCALE_HI_C == 1000;
+ assert K_PRESET_SCALE_C[1] > 448;
+ assert K_PRESET_SCALE_C[0] < 448;
+}
+
+test the_axis_holds_the_cap {
+ assert K_Y_HI_U > 10;
+ assert K_Y_TICKS_U[4] == K_Y_HI_U;
+ assert K_PRESET_SCALE_C[2] == K_SCALE_LO_C;
+}
diff --git a/apps/website/specs/widgets/nn-lstm.t27 b/apps/website/specs/widgets/nn-lstm.t27
new file mode 100644
index 0000000000..aeaa04a2ba
--- /dev/null
+++ b/apps/website/specs/widgets/nn-lstm.t27
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-lstm.t27 -- an LSTM cell's gates: forget, input, output
+; Source of truth for public/widgets/nn-lstm/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-lstm/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/recurrent/lstm_single.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 23 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets four gate biases from -6 to 6 and the old memory from -2 to 2. With
+; every weight 0, each gate is sigmoid(bias); the new memory and output are bias_only(b_i, b_f, b_g,
+; b_o, c_prev, which) of the spec.
+
+; WHAT IT DOES NOT CLAIM: one step of a 1-unit cell with all weights 0, so the gates see only their
+; biases; that is the spec's own fixture.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_lstm;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-lstm";
+pub const TITLE : str = "LSTM gates: how a cell decides what to keep";
+pub const DESCRIPTION : str = "Set the biases of the forget, input and output gates and the old memory: the new memory is forget x old + input x candidate, and the output shows tanh of it through the output gate.";
+pub const IMAGE_ALT : str = "LSTM card, remember: gate bars forget 0.998, input 0.002 and output 0.998; memory bars old 1.000, new 0.998 and output 0.759. Tiles read new memory 0.9975, output 0.7587, forget 0.9975 and input 0.0025.";
+pub const HOOK : str = "With the forget gate open (0.9975) and the input gate shut, a memory of 1 comes out as 0.9975: an LSTM can hold a value for a long time.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/recurrent/lstm_single.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 23 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_BIAS_HI_C : u16 = 600;
+pub const K_C_LO_C : i16 = -200;
+pub const K_C_HI_C : u16 = 200;
+pub const K_CELL_LO : i8 = -3;
+pub const K_CELL_HI : u8 = 3;
+pub const K_GATE_TICKS_C : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_CELL_TICKS : [5]i8 = [-2, -1, 0, 1, 2];
+pub const K_PRESET_BF_C : [4]i16 = [600, -600, -600, 600];
+pub const K_PRESET_BI_C : [4]i16 = [-600, 0, 600, -600];
+pub const K_PRESET_BG_C : [4]i16 = [0, 0, 600, 0];
+pub const K_PRESET_BO_C : [4]i16 = [600, 600, 600, -600];
+pub const K_PRESET_C_C : [4]i16 = [100, 100, 100, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Remember", "Forget", "Write", "Hide"];
+pub const SAY_ASK : str = "How does a memory cell decide what to keep and what to forget?";
+pub const SAY_BF : str = "forget bias";
+pub const SAY_BI : str = "input bias";
+pub const SAY_BG : str = "candidate bias";
+pub const SAY_BO : str = "output bias";
+pub const SAY_C_PREV : str = "old memory c";
+pub const SAY_GATES_LABEL : str = "The three gates, each a sigmoid of its bias";
+pub const SAY_CELL_LABEL : str = "The memory before and after, and the output";
+pub const SAY_FORGET : str = "forget";
+pub const SAY_INPUT : str = "input";
+pub const SAY_OUTPUT : str = "output";
+pub const SAY_C_OLD : str = "old c";
+pub const SAY_C_NEW : str = "new c";
+pub const SAY_H_NEW : str = "output h";
+pub const SAY_NOTE : str = "c' = f c + i g and h' = o tanh(c'). When f is near 1 and i near 0, c passes through almost untouched, and so does its gradient: the long memory the plain cell lacked.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/recurrent/lstm_single.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_biases_are_symmetric {
+ assert K_BIAS_HI_C == 600;
+ assert K_PRESET_BF_C[1] + K_BIAS_HI_C == 0;
+ assert K_PRESET_BG_C[2] == K_BIAS_HI_C;
+}
+
+test the_presets_cover_the_gates {
+ assert K_PRESET_BF_C[0] > 0;
+ assert K_PRESET_BI_C[0] < 0;
+ assert K_PRESET_BO_C[3] < 0;
+ assert K_PRESET_C_C[0] <= K_C_HI_C;
+}
diff --git a/apps/website/specs/widgets/nn-maxpool.t27 b/apps/website/specs/widgets/nn-maxpool.t27
new file mode 100644
index 0000000000..0cfa3a452c
--- /dev/null
+++ b/apps/website/specs/widgets/nn-maxpool.t27
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-maxpool.t27 -- max pooling: keep the largest value of each window, and where it was
+; Source of truth for public/widgets/nn-maxpool/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-maxpool/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/layers/maxpool2d_layer.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 11 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader picks one of four window and stride pairs and taps an output. Outputs and
+; argmax indices are pool_at(kernel, stride, 0 or 1, i) and the output side is out_side(kernel,
+; stride); for 2 x 2 stride 2 the gradient view is backward_at(i), the spec's output gradients
+; 1, 2, 3, 4.
+
+; WHAT IT DOES NOT CLAIM: the grid is the spec's fixture, repeated as K_INPUT; the outputs and
+; argmax come from the compiled fn, so a K_INPUT that drifted would show a max that is not in its
+; window.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_maxpool;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-maxpool";
+pub const TITLE : str = "Max pooling: what a network keeps when it shrinks a picture";
+pub const DESCRIPTION : str = "Pick a window size and stride and tap an output: max pooling keeps the largest value of its window and remembers where it was, and the gradient later goes to that one cell.";
+pub const IMAGE_ALT : str = "Max-pool card, 2 x 2 windows with stride 2: the spec's 4 x 4 grid in pink and green, with cell 13 (value 9) outlined inside its window; the 2 x 2 output reads 4, 8, 9, -2 with 9 outlined. Tiles read max 9, argmax 13, output 2 x 2 and gradient at the max 3.";
+pub const HOOK : str = "On the spec's 4 x 4 grid, 2 x 2 windows with stride 2 keep 4, 8, 9 and -2; their gradients land on cells 5, 6, 13 and 11 and nowhere else.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/layers/maxpool2d_layer.t27 (t27 master 2e53089d5): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 11 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SIDE : u8 = 4;
+pub const K_INPUT : [16]i8 = [1, 2, 5, 6, 3, 4, 8, 7, 0, -1, -3, -2, -4, 9, -6, -5];
+pub const K_KERNELS : [4]u8 = [2, 3, 2, 4];
+pub const K_STRIDES : [4]u8 = [2, 1, 1, 1];
+pub const K_PRESET_CFG : [3]u8 = [0, 1, 2];
+pub const K_PRESET_CELL : [3]u8 = [2, 0, 4];
+pub const K_PRESET_GRAD : [3]u8 = [0, 0, 0];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [3]str = ["2 x 2, stride 2", "3 x 3, stride 1", "2 x 2, stride 1"];
+pub const SAY_ASK : str = "What does a network keep when it shrinks a picture?";
+pub const SAY_CONFIG : str = "window, stride";
+pub const SAY_CONFIG_NAMES : [4]str = ["2 x 2 / 2", "3 x 3 / 1", "2 x 2 / 1", "4 x 4 / 1"];
+pub const SAY_SHOW_GRAD : str = "show the gradient (2 x 2 / 2)";
+pub const SAY_IN_LABEL : str = "The spec's 4 x 4 input, the max of the window outlined";
+pub const SAY_GRAD_LABEL : str = "The gradient each input receives";
+pub const SAY_OUT_LABEL : str = "The output: tap a cell";
+pub const SAY_MAX : str = "max";
+pub const SAY_ARGMAX : str = "from cell";
+pub const SAY_OUT_SIDE : str = "output";
+pub const SAY_GRAD_AT_MAX : str = "gradient there";
+pub const SAY_NO_GRAD : str = "2 x 2 / 2 only";
+pub const SAY_NOTE : str = "A max pool forgets every value but one per window, which makes the network care less about exactly where a feature sits. In backpropagation the whole gradient of a window goes to its maximum.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/layers/maxpool2d_layer.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_four_configs_are_the_spec_window_stride_pairs {
+ assert K_KERNELS[0] == 2;
+ assert K_STRIDES[0] == 2;
+ assert K_KERNELS[1] == 3;
+ assert K_STRIDES[1] == 1;
+ assert K_KERNELS[2] == 2;
+ assert K_STRIDES[2] == 1;
+ assert K_KERNELS[3] == K_SIDE;
+ assert K_STRIDES[3] == 1;
+}
+
+test the_grid_is_the_spec_fixture {
+ assert K_INPUT[5] == 4;
+ assert K_INPUT[6] == 8;
+ assert K_INPUT[13] == 9;
+ assert K_INPUT[11] + 2 == 0;
+}
diff --git a/apps/website/specs/widgets/nn-mse.t27 b/apps/website/specs/widgets/nn-mse.t27
new file mode 100644
index 0000000000..1f045e1fbd
--- /dev/null
+++ b/apps/website/specs/widgets/nn-mse.t27
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-mse.t27 -- mean squared error: how wrong a guess is, as one number
+; Source of truth for public/widgets/nn-mse/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-mse/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/loss/mse_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 13 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader moves three predictions from -1 to 5. The loss is mse_at(p0, p1, p2)
+; against the spec's targets (1, 2, 3), each gradient mse_grad_at, and the bowl mse_at over p0.
+
+; WHAT IT DOES NOT CLAIM: the targets are the spec's fixture, repeated as K_TARGETS; every loss and
+; gradient comes from the compiled fn.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_mse;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-mse";
+pub const TITLE : str = "Mean squared error: how wrong a guess is, as one number";
+pub const DESCRIPTION : str = "Move three predictions against the targets 1, 2 and 3: the loss is the mean of the squared misses, its bowl is a parabola, and each gradient points its prediction back at the target.";
+pub const IMAGE_ALT : str = "MSE card: predictions 1.5, 2 and 2 against targets 1, 2, 3 drawn as white rings with green dots, red dashed misses and pink gradient arrows. Below, the loss as prediction 0 moves is a gold parabola with its cursor at 1.5. Tiles read loss 0.4167, gradients 0.333 0.000 -0.667, loss at the targets 0.0000 and misses of 1 and 2 costing 0.333 and 1.333.";
+pub const HOOK : str = "A miss of 1 costs 1 / 3 of the mean; a miss of 2 costs 4 / 3. Squaring is what makes MSE fear large errors and shrug at small ones.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/loss/mse_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 13 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_TARGETS : [3]u8 = [1, 2, 3];
+pub const K_P_LO_C : i16 = -100;
+pub const K_P_HI_C : u16 = 500;
+pub const K_P_TICKS : [7]i8 = [-1, 0, 1, 2, 3, 4, 5];
+pub const K_LOSS_HI_C : u16 = 600;
+pub const K_LOSS_TICKS : [4]u8 = [0, 2, 4, 6];
+pub const K_PRESET_P0_C : [4]i16 = [150, 100, 300, -100];
+pub const K_PRESET_P1_C : [4]u16 = [200, 200, 200, 400];
+pub const K_PRESET_P2_C : [4]u16 = [200, 300, 300, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["1.5, 2, 2", "On target", "One miss of 2", "All wrong"];
+pub const SAY_ASK : str = "How wrong is a guess, as one number?";
+pub const SAY_PRED : str = "prediction";
+pub const SAY_TARGET : str = "target";
+pub const SAY_GRAD : str = "minus the gradient";
+pub const SAY_MISS_LABEL : str = "Each prediction, its target and the step the gradient asks for";
+pub const SAY_BOWL_LABEL : str = "The loss as prediction 0 moves, the others held";
+pub const SAY_BOWL : str = "Drag along the bowl: the loss as prediction 0 alone moves.";
+pub const SAY_LOSS : str = "loss";
+pub const SAY_GRADS : str = "gradients";
+pub const SAY_AT_TARGET : str = "loss on target";
+pub const SAY_DOUBLE : str = "miss of 1 / of 2";
+pub const SAY_NOTE : str = "The gradient of each prediction is 2 (p - t) / n: the further off, the harder the pull. Gradient descent follows it downhill into the bottom of the bowl.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/loss/mse_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_targets_are_the_spec_fixture {
+ assert K_TARGETS[0] == 1;
+ assert K_TARGETS[1] == 2;
+ assert K_TARGETS[2] == 3;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_P0_C[3] >= K_P_LO_C;
+ assert K_PRESET_P1_C[3] <= K_P_HI_C;
+ assert K_PRESET_P0_C[1] == K_TARGETS[0] * 100;
+}
diff --git a/apps/website/specs/widgets/nn-positions.t27 b/apps/website/specs/widgets/nn-positions.t27
new file mode 100644
index 0000000000..1d38dcee64
--- /dev/null
+++ b/apps/website/specs/widgets/nn-positions.t27
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-positions.t27 -- sinusoidal positional encoding: stripes and clocks
+; Source of truth for public/widgets/nn-positions/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-positions/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/transformer/positional_encoding.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 25 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the position from 0 to 200. Every stripe, hand and value is pe(pos,
+; dim) of the spec at d_model 8; the speeds are inv_freq(pair, 8).
+
+; WHAT IT DOES NOT CLAIM: d_model 8 shows the pattern; a real model uses hundreds of dimensions, and
+; many now learn or rotate positions instead.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_positions;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-positions";
+pub const TITLE : str = "Positional encoding: how a transformer knows word order";
+pub const DESCRIPTION : str = "Every position gets 8 numbers, sines and cosines at frequencies falling by 10 per pair. Drag the position: the fast clock spins, the slow ones barely move, and the stripes show all 48 rows.";
+pub const IMAGE_ALT : str = "Positional-encoding card at position 5: four clocks with coloured hands, the fastest turned far round, the slowest still pointing up. Below, a stripe chart of 48 positions by 8 dimensions in pink and green: the left columns flicker fast, the right ones barely change. Tiles read the 8 values and the 4 speeds 1, 0.1, 0.01, 0.001.";
+pub const HOOK : str = "Position 5 is (-0.959, 0.284, 0.479, 0.878, 0.050, 0.999, 0.005, 1.000). Four clocks at speeds 1, 0.1, 0.01 and 0.001 give each position its own reading.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/transformer/positional_encoding.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 25 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_D_MODEL : u8 = 8;
+pub const K_ROWS : u8 = 48;
+pub const K_POS_HI : u8 = 200;
+pub const K_PRESET_POS : [4]u8 = [5, 0, 1, 37];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Position 5", "Position 0", "Position 1", "Position 37"];
+pub const SAY_ASK : str = "Attention sees a set of words. How does it know their order?";
+pub const SAY_POS : str = "position";
+pub const SAY_STRIPES_LABEL : str = "pe(pos, dim) for positions 0 to 47 (rows) and the 8 dimensions";
+pub const SAY_CLOCKS_LABEL : str = "Each sin and cos pair as a clock hand, and its speed";
+pub const SAY_PAIR : str = "pair";
+pub const SAY_VECTOR : str = "the 8 numbers";
+pub const SAY_FAST : str = "fastest pair";
+pub const SAY_SLOW : str = "slowest pair";
+pub const SAY_FREQS : str = "speeds";
+pub const SAY_NOTE : str = "A shift by k positions turns every clock by its own fixed angle, so the distance between two positions is visible to a dot product. Position 0 is (0, 1, 0, 1, ...).";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/transformer/positional_encoding.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test eight_dimensions_four_clocks {
+ assert K_D_MODEL == 8;
+ assert K_D_MODEL % 2 == 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_POS[3] < K_ROWS;
+ assert K_PRESET_POS[1] == 0;
+ assert K_POS_HI > K_ROWS;
+}
diff --git a/apps/website/specs/widgets/nn-ppo.t27 b/apps/website/specs/widgets/nn-ppo.t27
new file mode 100644
index 0000000000..f32a2aeb05
--- /dev/null
+++ b/apps/website/specs/widgets/nn-ppo.t27
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-ppo.t27 -- PPO's clipped surrogate: a reward that stops pulling
+; Source of truth for public/widgets/nn-ppo/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-ppo/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/rl/ppo_clip_loss.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 27 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags r from 0 to 2 and sets the advantage A from -2 to 2 and eps from
+; 0.05 to 0.5. The green curve is clipped_surrogate(r, A, eps), the dashed one clipped_surrogate(r,
+; A, 1000), which never clips here, and the tiles policy_loss_of on the spec's advantages (2, 1.5,
+; 3).
+
+; WHAT IT DOES NOT CLAIM: the objective of one sample; PPO averages it over a batch, adds value and
+; entropy terms (total_loss in the spec) and runs it for a few epochs per batch.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_ppo;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-ppo";
+pub const TITLE : str = "PPO: how an agent learns from reward without lurching";
+pub const DESCRIPTION : str = "Drag the probability ratio of a new policy over the old: PPO's objective follows r times the advantage until r moves more than eps the rewarded way, then it goes flat and stops pulling.";
+pub const IMAGE_ALT : str = "PPO card at r = 1.5, A = 1, eps 0.2: the dashed gold raw objective is a straight line through the origin; the thick green clipped one follows it and turns flat at 1.2 from r = 1.2 on. Tiles read clipped 1.2000, raw 1.5000, the spec batch policy loss -2.1167 and the same batch at r = 1.5, -2.6000.";
+pub const HOOK : str = "At r = 1.5 with advantage 1 and eps 0.2, the raw objective is 1.5 but PPO counts only 1.2. The clip is what keeps one update from breaking the policy.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/rl/ppo_clip_loss.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 27 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_R_HI_C : u16 = 200;
+pub const K_A_LO_C : i16 = -200;
+pub const K_A_HI_C : u16 = 200;
+pub const K_EPS_LO_C : u8 = 5;
+pub const K_EPS_HI_C : u8 = 50;
+pub const K_WIDE : u16 = 1000;
+pub const K_Y_LO : i8 = -4;
+pub const K_Y_HI : u8 = 4;
+pub const K_Y_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_R_TICKS_C : [5]u8 = [0, 50, 100, 150, 200];
+pub const K_BATCH_R_C : [3]u8 = [100, 110, 90];
+pub const K_PRESET_R_C : [4]u8 = [150, 50, 150, 80];
+pub const K_PRESET_A_C : [4]i16 = [100, -100, 200, -200];
+pub const K_PRESET_EPS_C : [4]u8 = [20, 20, 50, 10];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["r 1.5, A 1", "r 0.5, A -1", "r 1.5, A 2, eps 0.5", "r 0.8, A -2, eps 0.1"];
+pub const SAY_ASK : str = "How does an agent learn from reward without lurching?";
+pub const SAY_R : str = "ratio r = new / old";
+pub const SAY_A : str = "advantage A";
+pub const SAY_EPS : str = "clip eps";
+pub const SAY_PLOT_LABEL : str = "PPO's clipped objective and the raw r A as the ratio goes from 0 to 2";
+pub const SAY_CLIPPED : str = "PPO objective";
+pub const SAY_RAW : str = "raw r A";
+pub const SAY_BATCH : str = "spec batch loss";
+pub const SAY_BATCH_FAR : str = "same batch at r";
+pub const SAY_NOTE : str = "A positive advantage says the action was better than expected, so raise its probability; past 1 + eps there is no more credit for it. A negative one is clipped at 1 - eps the same way.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/rl/ppo_clip_loss.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_wide_clip_never_clips_the_window {
+ assert K_WIDE * 100 > K_R_HI_C;
+ assert K_EPS_HI_C < 100;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_R_C[0] <= K_R_HI_C;
+ assert K_PRESET_A_C[3] == K_A_LO_C;
+ assert K_PRESET_EPS_C[3] >= K_EPS_LO_C;
+ assert K_BATCH_R_C[0] == 100;
+}
diff --git a/apps/website/specs/widgets/nn-relu.t27 b/apps/website/specs/widgets/nn-relu.t27
new file mode 100644
index 0000000000..b63bcb47e7
--- /dev/null
+++ b/apps/website/specs/widgets/nn-relu.t27
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-relu.t27 -- ReLU and leaky ReLU: what a neuron does with a negative number
+; Source of truth for public/widgets/nn-relu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-relu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/relu_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 1 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -4 to 4 and sets a leaky slope from 0 to 0.3. The green
+; curve is forward_at over x and the dashed curve backward_at, the slope; the tiles read both at x
+; and over the spec's own three-input batch (-2, 0, 2).
+
+; WHAT IT DOES NOT CLAIM: ReLU is one choice among the activations of module 2; the page measures
+; nothing about training.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_relu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-relu";
+pub const TITLE : str = "ReLU: what a neuron does with a negative number";
+pub const DESCRIPTION : str = "Drag along the curve and watch ReLU pass every positive input and zero every negative one; give it a leaky slope and a little of the negative side gets through.";
+pub const IMAGE_ALT : str = "ReLU card: leaky slope 0.1 at x = -2. The green curve is flat at -0.2 on the left and rises at 45 degrees on the right; the dashed gold slope sits at 0.1 then jumps to 1. Tiles read out -0.200, slope 0.100 and the spec's batch (-2, 0, 2) as -0.20 0.00 2.00.";
+pub const HOOK : str = "max(0, x) is the whole rule, and its slope is 1 or 0. A leaky slope of 0.1 keeps a gradient alive on the negative side.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/relu_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 1 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -400;
+pub const K_X_HI_C : i16 = 400;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_Y_LO_C : i16 = -100;
+pub const K_Y_HI_C : i16 = 400;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [6]i8 = [-1, 0, 1, 2, 3, 4];
+pub const K_SLOPE_MAX_C : u8 = 30;
+pub const K_FIXTURE_C : [3]i16 = [-200, 0, 200];
+pub const K_PRESET_X_C : [4]i16 = [-200, 150, -200, -300];
+pub const K_PRESET_SLOPE_C : [4]u8 = [10, 0, 0, 30];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Leaky 0.1 at x = -2", "ReLU at x = 1.5", "ReLU at x = -2", "Leaky 0.3 at x = -3"];
+pub const SAY_ASK : str = "What does a neuron do with a negative number?";
+pub const SAY_X : str = "input x";
+pub const SAY_SLOPE : str = "leaky slope";
+pub const SAY_F : str = "ReLU(x)";
+pub const SAY_DF : str = "slope dReLU/dx";
+pub const SAY_PLOT_LABEL : str = "ReLU of x and its slope from -4 to 4, with a cursor you drag";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_BATCH : str = "spec batch -2 0 2";
+pub const SAY_BATCH_GRAD : str = "its slopes";
+pub const SAY_NOTE : str = "A slope of 0 means no gradient flows back, so a unit that only ever sees negative inputs stops learning. The leaky slope is the fix the spec takes as its fourth argument.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/relu_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_range_holds_both_sides {
+ assert K_X_LO_C < 0;
+ assert K_X_HI_C > 0;
+ assert K_X_LO_C + K_X_HI_C == 0;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_X_C[3] >= K_X_LO_C;
+ assert K_PRESET_SLOPE_C[3] <= K_SLOPE_MAX_C;
+ assert K_PRESET_SLOPE_C[1] == 0;
+ assert K_FIXTURE_C[0] == K_PRESET_X_C[0];
+}
diff --git a/apps/website/specs/widgets/nn-rmsprop.t27 b/apps/website/specs/widgets/nn-rmsprop.t27
new file mode 100644
index 0000000000..1ee610febf
--- /dev/null
+++ b/apps/website/specs/widgets/nn-rmsprop.t27
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-rmsprop.t27 -- RMSProp: a step that ignores the size of the gradient
+; Source of truth for public/widgets/nn-rmsprop/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-rmsprop/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/rmsprop.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 17 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader picks a gradient, a decay from 0.5 to 0.99 and a number of steps. Every
+; point is param_after(g, n, decay) from 1.0 with lr 0.01, the cache cache_after(g, n) at the spec's
+; decay 0.9.
+
+; WHAT IT DOES NOT CLAIM: a constant gradient on one weight; with a changing gradient the cache lags
+; behind it.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_rmsprop;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-rmsprop";
+pub const TITLE : str = "RMSProp: a step that does not care how big the gradient is";
+pub const DESCRIPTION : str = "Feed one weight a constant gradient of 0.001, 0.5 or 1000: RMSProp divides by the root of its running mean square, so all three walk the same path. Change the decay and the path bends.";
+pub const IMAGE_ALT : str = "RMSProp card: gradient 0.5, decay 0.9, 100 steps. Three lines, green for 0.5 and dashed gold and pink for 0.001 and 1000, lie exactly on top of each other, falling from 1 to about -0.09. Tiles read the weight -0.09142, the three runs -0.0914 -0.0914 -0.0914, one step 0.96838 and the cache 0.249993.";
+pub const HOOK : str = "Its first step is 0.0316 whether the gradient is 0.001 or 1000. Dividing by the gradient's own size is what makes the step scale-free.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/rmsprop.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 17 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS_HI : u8 = 100;
+pub const K_GRADS_M : [3]u32 = [1, 500, 1000000];
+pub const K_DECAY_LO_C : u8 = 50;
+pub const K_DECAY_HI_C : u8 = 99;
+pub const K_Y_LO_C : i16 = -110;
+pub const K_Y_HI_C : u8 = 110;
+pub const K_Y_TICKS_C : [5]i8 = [-100, -50, 0, 50, 100];
+pub const K_STEP_TICKS : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_G : [4]u8 = [1, 0, 2, 1];
+pub const K_PRESET_DECAY_C : [4]u8 = [90, 90, 90, 99];
+pub const K_PRESET_STEPS : [4]u8 = [100, 100, 100, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["g = 0.5", "g = 0.001", "g = 1000", "g = 0.5, decay 0.99"];
+pub const SAY_ASK : str = "Can a step ignore how big the gradient is?";
+pub const SAY_GRAD : str = "gradient";
+pub const SAY_GRAD_NAMES : [3]str = ["g = 0.001", "g = 0.5", "g = 1000"];
+pub const SAY_DECAY : str = "decay";
+pub const SAY_STEPS : str = "steps";
+pub const SAY_PLOT_LABEL : str = "The weight after every step for three gradient sizes";
+pub const SAY_W_NOW : str = "weight now";
+pub const SAY_W_OTHERS : str = "all three gradients";
+pub const SAY_ONE_STEP : str = "after step 1";
+pub const SAY_CACHE : str = "cache, decay 0.9";
+pub const SAY_NOTE : str = "The cache is a running mean of g squared, so g / sqrt(cache) has no units: a layer whose gradients are tiny moves as fast as one whose gradients are huge.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/rmsprop.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_gradients_span_six_decades {
+ assert K_GRADS_M[0] == 1;
+ assert K_GRADS_M[2] == 1000000;
+ assert K_GRADS_M[1] == 500;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_G[2] < 3;
+ assert K_PRESET_DECAY_C[3] == K_DECAY_HI_C;
+ assert K_PRESET_STEPS[0] == K_STEPS_HI;
+}
diff --git a/apps/website/specs/widgets/nn-rnn.t27 b/apps/website/specs/widgets/nn-rnn.t27
new file mode 100644
index 0000000000..2335230222
--- /dev/null
+++ b/apps/website/specs/widgets/nn-rnn.t27
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-rnn.t27 -- a recurrent cell: an echo that fades
+; Source of truth for public/widgets/nn-rnn/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-rnn/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/recurrent/rnn_cell.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 22 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader taps each of eight inputs to cycle it through -1, 0 and +1. Each h is
+; tiny_step(x, h) of the spec, applied in order from h = 0; the spec's own three-step run is
+; tiny_sequence(t).
+
+; WHAT IT DOES NOT CLAIM: a 1-in, 1-hidden cell with the spec's weights (W_x 1, W_h 0.5, b 0); a
+; real cell has a vector of hidden units.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_rnn;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-rnn";
+pub const TITLE : str = "Recurrent cells: how a network remembers a moment ago";
+pub const DESCRIPTION : str = "Tap the inputs of eight steps: the cell feeds its own last output back, h' = tanh(x + 0.5 h), so every pulse echoes on and fades by about half a step. Two pulses add up.";
+pub const IMAGE_ALT : str = "RNN card: inputs +1, 0, 0, +1, 0, 0, 0, 0 as cyan bars, and the green hidden state jumping to 0.76, fading to 0.36 and 0.18, jumping to 0.80 at the second pulse and fading to 0.05. Tiles read the last h 0.0466 and the spec's own run 0.7616 0.3634 0.1797.";
+pub const HOOK : str = "A single pulse gives 0.762, 0.363, 0.180: the memory halves each step. A cell remembers, but not for long, which is the problem LSTMs solve.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/recurrent/rnn_cell.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 22 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS : u8 = 8;
+pub const K_Y_TICKS : [3]i8 = [-1, 0, 1];
+pub const K_PRESET_INPUTS : [32]i8 = [1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 1, -1, 1, -1, 1, -1, 1, 1, 1, 1, 1, 1, 1, 1];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Two pulses", "One pulse", "Alternating", "Always on"];
+pub const SAY_ASK : str = "How does a network remember what it saw a moment ago?";
+pub const SAY_TAP : str = "Inputs: tap a step to change it";
+pub const SAY_STEP : str = "step";
+pub const SAY_X : str = "input x";
+pub const SAY_H : str = "hidden state h";
+pub const SAY_PLOT_LABEL : str = "Input and hidden state at every step";
+pub const SAY_H_LAST : str = "h, last step";
+pub const SAY_H_ALL : str = "h at every step";
+pub const SAY_SPEC_RUN : str = "spec run 1, 0, 0";
+pub const SAY_ECHO : str = "echo of one pulse";
+pub const SAY_NOTE : str = "The same weights act at every step, and the state h carries what came before. Because 0.5 h is squeezed by tanh each time, an old input fades geometrically.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/recurrent/rnn_cell.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test eight_steps_four_presets {
+ assert K_STEPS == 8;
+ assert K_PRESET_INPUTS[0] == 1;
+ assert K_PRESET_INPUTS[3] == 1;
+}
+
+test the_inputs_are_trits {
+ assert K_PRESET_INPUTS[17] + 1 == 0;
+ assert K_PRESET_INPUTS[31] == 1;
+ assert K_PRESET_INPUTS[8] == 1;
+ assert K_PRESET_INPUTS[9] == 0;
+}
diff --git a/apps/website/specs/widgets/nn-sgd.t27 b/apps/website/specs/widgets/nn-sgd.t27
new file mode 100644
index 0000000000..6efd6e901e
--- /dev/null
+++ b/apps/website/specs/widgets/nn-sgd.t27
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-sgd.t27 -- gradient descent with momentum and Nesterov
+; Source of truth for public/widgets/nn-sgd/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-sgd/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/optimizer/sgd.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 16 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets lr from 0.01 to 1, momentum from 0 to 0.95, Nesterov and the number
+; of steps. Every point is descend(lr, momentum, nesterov, n), the spec minimising (x - 3)^2 from 0;
+; the dashed path is descend with momentum 0.
+
+; WHAT IT DOES NOT CLAIM: one weight on one bowl; a network has millions and its bowl is not a
+; parabola.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_sgd;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-sgd";
+pub const TITLE : str = "Gradient descent: how a network walks downhill";
+pub const DESCRIPTION : str = "Set the learning rate and momentum and watch a weight walk from 0 to the bottom of (x - 3)^2. Plain steps creep, momentum overshoots and rings, and a rate of 1 bounces forever.";
+pub const IMAGE_ALT : str = "SGD card: momentum 0.9 at lr 0.1 for 50 steps. The green path of dots shoots up from 0 past the white dashed minimum at 3 to about 5.1, swings back below it and rings in toward 3; the dashed violet plain path creeps up smoothly. Tiles read x after 50 steps 3.09149, plain 2.99996 and step 1 at 0.6000.";
+pub const HOOK : str = "lr 0.1 reaches 2.99996 in 50 steps. Add momentum 0.9 and it shoots to 5.13 first; at lr 1 it bounces between 0 and 6 for good.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/optimizer/sgd.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 16 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_STEPS_HI : u8 = 100;
+pub const K_MINIMUM : u8 = 3;
+pub const K_LR_LO_C : u8 = 1;
+pub const K_LR_HI_C : u8 = 100;
+pub const K_MOM_HI_C : u8 = 95;
+pub const K_Y_LO : i8 = -1;
+pub const K_Y_HI : u8 = 7;
+pub const K_Y_TICKS : [5]i8 = [0, 2, 3, 4, 6];
+pub const K_STEP_TICKS : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_LR_C : [4]u8 = [10, 10, 10, 100];
+pub const K_PRESET_MOM_C : [4]u8 = [90, 0, 90, 0];
+pub const K_PRESET_NEST : [4]u8 = [0, 0, 1, 0];
+pub const K_PRESET_STEPS : [4]u8 = [50, 50, 50, 20];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Momentum 0.9", "Plain, lr 0.1", "Nesterov 0.9", "lr 1: it bounces"];
+pub const SAY_ASK : str = "How does a network walk downhill, and why does it overshoot?";
+pub const SAY_LR : str = "learning rate";
+pub const SAY_MOMENTUM : str = "momentum";
+pub const SAY_NESTEROV : str = "Nesterov";
+pub const SAY_STEPS : str = "steps";
+pub const SAY_PLOT_LABEL : str = "The weight after every step, from 0 toward the minimum at 3";
+pub const SAY_PATH : str = "this run";
+pub const SAY_PLAIN : str = "no momentum";
+pub const SAY_MIN : str = "the minimum";
+pub const SAY_X_NOW : str = "x after the steps";
+pub const SAY_X_PLAIN : str = "without momentum";
+pub const SAY_X_ONE : str = "after step 1";
+pub const SAY_NOTE : str = "Each step moves x by -lr times the gradient 2 (x - 3). Momentum adds 0.9 of the last move, which speeds up a long slope and overshoots a sharp bottom; Nesterov takes the gradient one step ahead.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/optimizer/sgd.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_minimum_is_inside_the_axis {
+ assert K_MINIMUM == 3;
+ assert K_Y_LO < 0;
+ assert K_Y_HI > K_MINIMUM * 2;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_LR_C[3] == K_LR_HI_C;
+ assert K_PRESET_MOM_C[0] <= K_MOM_HI_C;
+ assert K_PRESET_STEPS[0] <= K_STEPS_HI;
+}
diff --git a/apps/website/specs/widgets/nn-sigmoid.t27 b/apps/website/specs/widgets/nn-sigmoid.t27
new file mode 100644
index 0000000000..f88b67dbf7
--- /dev/null
+++ b/apps/website/specs/widgets/nn-sigmoid.t27
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-sigmoid.t27 -- the sigmoid: any number into a probability
+; Source of truth for public/widgets/nn-sigmoid/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-sigmoid/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/sigmoid_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 2 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -8 to 8. The curve is forward and the slope derivative of
+; the spec, which takes the OUTPUT of the sigmoid, as backpropagation does; the tiles read the slope
+; at x, at 0 and at the far right.
+
+; WHAT IT DOES NOT CLAIM: the slope bound of 0.25 is read off the compiled curve at x = 0, not
+; proved here.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_sigmoid;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-sigmoid";
+pub const TITLE : str = "Sigmoid: turn any number into a probability";
+pub const DESCRIPTION : str = "Drag along the S-curve: every input lands between 0 and 1, and the slope peaks at 0.25 in the middle and dies away at both ends.";
+pub const IMAGE_ALT : str = "Sigmoid card at x = 2: the green S-curve from 0 to 1 over -8..8, the dashed gold slope a bump peaking at 0.25 at x = 0. Tiles read sigmoid 0.8808, slope 0.1050, slope at 0 = 0.2500 and slope at 8 = 0.000335; two bars repeat the output and the slope.";
+pub const HOOK : str = "1 / (1 + e^-x) squeezes the whole line into 0..1. Its slope is never above 0.25, which is why deep sigmoid nets learned slowly.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/sigmoid_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 2 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -800;
+pub const K_X_HI_C : i16 = 800;
+pub const K_X_STEP_C : u8 = 10;
+pub const K_X_TICKS : [5]i8 = [-8, -4, 0, 4, 8];
+pub const K_Y_TICKS_C : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_X_C : [4]i16 = [200, 0, -400, 600];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["x = 2", "x = 0", "x = -4", "x = 6"];
+pub const SAY_ASK : str = "How does a neuron turn any number into a probability?";
+pub const SAY_X : str = "input x";
+pub const SAY_F : str = "sigmoid(x)";
+pub const SAY_DF : str = "slope";
+pub const SAY_PLOT_LABEL : str = "The sigmoid of x and its slope from -8 to 8, with a cursor you drag";
+pub const SAY_BARS_LABEL : str = "The output and the slope at x as bars from 0 to 1";
+pub const SAY_OUT : str = "probability";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_AT_ZERO : str = "slope at 0";
+pub const SAY_FAR : str = "slope at 8";
+pub const SAY_NOTE : str = "The slope is s (1 - s), computed from the output s itself. Far from 0 it is close to nothing, so a saturated sigmoid passes almost no gradient back.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/sigmoid_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_range_is_symmetric {
+ assert K_X_LO_C + K_X_HI_C == 0;
+ assert K_X_HI_C == 800;
+}
+
+test the_axis_spans_zero_to_one {
+ assert K_Y_TICKS_C[0] == 0;
+ assert K_Y_TICKS_C[4] == 100;
+ assert K_PRESET_X_C[1] == 0;
+}
diff --git a/apps/website/specs/widgets/nn-silu.t27 b/apps/website/specs/widgets/nn-silu.t27
new file mode 100644
index 0000000000..087ee92a09
--- /dev/null
+++ b/apps/website/specs/widgets/nn-silu.t27
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-silu.t27 -- Swish and SiLU: one knob between a smooth curve and ReLU
+; Source of truth for public/widgets/nn-silu/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-silu/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/silu_swish_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 5 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -5 to 5 and sets beta from 0.25 to 8. The green curve is
+; forward(x, beta), the dotted one forward(x, 1), SiLU, and the dashed one derivative(x, beta).
+
+; WHAT IT DOES NOT CLAIM: the claim that large beta approaches ReLU is what the curves show on this
+; window, not a limit taken here.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_silu;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-silu";
+pub const TITLE : str = "Swish: one knob between a smooth curve and ReLU";
+pub const DESCRIPTION : str = "Drag along x times sigmoid(beta x). At beta 1 it is SiLU, with its dip of -0.28; turn beta up and the curve closes on ReLU, turn it down and it flattens toward x / 2.";
+pub const IMAGE_ALT : str = "Swish card at beta 1, x = -1.28: the green curve dips to -0.2785 and then rises along the diagonal, over a dotted violet SiLU reference that matches it; the dashed gold slope crosses zero at the cursor. Tiles read output -0.2785, slope -0.0003, SiLU -0.2785 and the spec batch (-2, 0, 2) as -0.238 0.000 1.762.";
+pub const HOOK : str = "SiLU(-1.28) = -0.2785, its lowest point. At beta 8 the dip shrinks to almost nothing and the curve is ReLU with rounded corners.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/silu_swish_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 5 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -500;
+pub const K_X_HI_C : i16 = 500;
+pub const K_X_STEP_C : u8 = 2;
+pub const K_Y_LO_C : i16 = -100;
+pub const K_Y_HI_C : i16 = 500;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [7]i8 = [-1, 0, 1, 2, 3, 4, 5];
+pub const K_BETA_LO_C : u8 = 25;
+pub const K_BETA_HI_C : u16 = 800;
+pub const K_BETA_STEP_C : u8 = 25;
+pub const K_FIXTURE_C : [3]i16 = [-200, 0, 200];
+pub const K_PRESET_X_C : [4]i16 = [-128, -128, 200, 200];
+pub const K_PRESET_BETA_C : [4]u16 = [100, 400, 50, 800];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["SiLU at its dip", "beta 4 at the same x", "beta 0.5 at x = 2", "beta 8 at x = 2"];
+pub const SAY_ASK : str = "Is there one knob between a smooth curve and ReLU?";
+pub const SAY_X : str = "input x";
+pub const SAY_BETA : str = "beta";
+pub const SAY_F : str = "x sigmoid(beta x)";
+pub const SAY_SILU : str = "SiLU, beta 1";
+pub const SAY_DF : str = "slope";
+pub const SAY_PLOT_LABEL : str = "Swish of x for the chosen beta, SiLU for reference, and the slope";
+pub const SAY_OUT : str = "output";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_SILU_AT : str = "SiLU at x";
+pub const SAY_BATCH : str = "spec batch -2 0 2";
+pub const SAY_NOTE : str = "Swish multiplies x by a sigmoid gate of itself. beta sets how sharp the gate is: the same unit can be gentle or close to a switch.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/silu_swish_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test beta_one_is_silu {
+ assert K_PRESET_BETA_C[0] == 100;
+ assert K_BETA_LO_C < K_PRESET_BETA_C[0];
+ assert K_PRESET_BETA_C[3] == K_BETA_HI_C;
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_X_C[0] >= K_X_LO_C;
+ assert K_PRESET_X_C[3] <= K_X_HI_C;
+ assert K_BETA_HI_C % K_BETA_STEP_C == 0;
+}
diff --git a/apps/website/specs/widgets/nn-softmax.t27 b/apps/website/specs/widgets/nn-softmax.t27
new file mode 100644
index 0000000000..09d52e7cde
--- /dev/null
+++ b/apps/website/specs/widgets/nn-softmax.t27
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-softmax.t27 -- softmax and temperature: scores into a choice
+; Source of truth for public/widgets/nn-softmax/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-softmax/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/softmax.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 7 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader sets three scores from -3 to 3 and a temperature from 0.1 to 5. The bars
+; are softmax_at(a, b, c, T, i) and the sweep draws the same fn over T, with a cursor to drag.
+
+; WHAT IT DOES NOT CLAIM: three scores stand for a layer of any width; the spec's softmax_at takes
+; three.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_softmax;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-softmax";
+pub const TITLE : str = "Softmax: how a network turns scores into a choice";
+pub const DESCRIPTION : str = "Move three scores and the temperature: softmax turns them into probabilities that sum to one. Cool it and the top score takes almost everything; heat it and the choice evens out.";
+pub const IMAGE_ALT : str = "Softmax card: scores 1, 2 and 3 at temperature 1. Three bars in cyan, pink and green stand at 0.0900, 0.2447 and 0.6652. Below, three curves sweep the temperature from 0.1 to 5: the green one falls from 1 toward 0.4 while the other two rise, with a cursor at T = 1.";
+pub const HOOK : str = "Scores 1, 2, 3 give 0.090, 0.245, 0.665. At temperature 0.5 the top one takes 0.867; at 5 the three are 0.27, 0.33, 0.40.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/softmax.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 7 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_SCORE_LO_C : i16 = -300;
+pub const K_SCORE_HI_C : i16 = 300;
+pub const K_T_LO_C : u8 = 10;
+pub const K_T_HI_C : u16 = 500;
+pub const K_T_STEP_C : u8 = 5;
+pub const K_T_TICKS : [6]u8 = [0, 1, 2, 3, 4, 5];
+pub const K_P_TICKS_C : [5]u8 = [0, 25, 50, 75, 100];
+pub const K_PRESET_A_C : [4]i16 = [100, 100, 100, 200];
+pub const K_PRESET_B_C : [4]i16 = [200, 200, 200, 200];
+pub const K_PRESET_C_C : [4]i16 = [300, 300, 300, 0];
+pub const K_PRESET_T_C : [4]u16 = [100, 50, 500, 100];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["1, 2, 3 at T = 1", "1, 2, 3 at T = 0.5", "1, 2, 3 at T = 5", "A tie: 2, 2, 0"];
+pub const SAY_ASK : str = "How does a network turn scores into a choice?";
+pub const SAY_A : str = "A";
+pub const SAY_B : str = "B";
+pub const SAY_C : str = "C";
+pub const SAY_T : str = "temperature T";
+pub const SAY_BARS_LABEL : str = "The three probabilities softmax gives the three scores";
+pub const SAY_SWEEP_LABEL : str = "Each probability as the temperature goes from 0.1 to 5";
+pub const SAY_SWEEP : str = "Drag along the temperature: the same three scores, cooled and heated.";
+pub const SAY_P : str = "p({0})";
+pub const SAY_COLD : str = "at T = {0}";
+pub const SAY_NOTE : str = "Softmax is exp(x / T) over the sum of all of them, with the largest score subtracted first so exp never overflows. A language model samples its next token from exactly this.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/softmax.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_temperature_stays_positive {
+ assert K_T_LO_C > 0;
+ assert K_T_HI_C == 500;
+ assert K_PRESET_T_C[1] < K_PRESET_T_C[0];
+}
+
+test the_presets_stay_in_range {
+ assert K_PRESET_C_C[0] == K_SCORE_HI_C;
+ assert K_PRESET_A_C[3] == K_PRESET_B_C[3];
+ assert K_PRESET_T_C[2] == K_T_HI_C;
+}
diff --git a/apps/website/specs/widgets/nn-tanh.t27 b/apps/website/specs/widgets/nn-tanh.t27
new file mode 100644
index 0000000000..e842bc0f11
--- /dev/null
+++ b/apps/website/specs/widgets/nn-tanh.t27
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-tanh.t27 -- tanh and the saturated band where learning stops
+; Source of truth for public/widgets/nn-tanh/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-tanh/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/activation/tanh_activation.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 3 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags x from -4 to 4. The curve is forward, the slope derivative, and
+; every sampled x whose slope is under K_SAT is shaded red.
+
+; WHAT IT DOES NOT CLAIM: the threshold 0.1 is a choice made for the picture, not a constant of the
+; spec.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_tanh;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-tanh";
+pub const TITLE : str = "tanh: where a neuron stops learning";
+pub const DESCRIPTION : str = "Drag along tanh: it is centred on zero, and where it flattens out its slope drops under 0.1. The red band shows every input where the gradient has all but stopped.";
+pub const IMAGE_ALT : str = "tanh card at x = 2.5: the green curve from -1 to 1, the dashed gold slope a bell peaking at 1, and two red bands on both sides where the slope is under 0.1. Tiles read tanh 0.9866, slope 0.0266 marked saturated, and the spec batch tanh(0, 1, 2) = 0 0.762 0.964.";
+pub const HOOK : str = "tanh(2.5) is already 0.987 and its slope 0.027. Past about 1.8 either way the gradient is under 0.1: the band a deep net must stay out of.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/activation/tanh_activation.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 3 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_X_LO_C : i16 = -400;
+pub const K_X_HI_C : i16 = 400;
+pub const K_X_STEP_C : u8 = 5;
+pub const K_X_TICKS : [5]i8 = [-4, -2, 0, 2, 4];
+pub const K_Y_TICKS : [3]i8 = [-1, 0, 1];
+pub const K_SAT_C : u8 = 10;
+pub const K_PRESET_X_C : [4]i16 = [250, 0, 100, -300];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["x = 2.5", "x = 0", "x = 1", "x = -3"];
+pub const SAY_ASK : str = "Where does a neuron stop learning?";
+pub const SAY_X : str = "input x";
+pub const SAY_F : str = "tanh(x)";
+pub const SAY_DF : str = "slope";
+pub const SAY_SAT_BAND : str = "slope under 0.1";
+pub const SAY_PLOT_LABEL : str = "tanh of x and its slope from -4 to 4, saturated inputs shaded";
+pub const SAY_OUT : str = "tanh(x)";
+pub const SAY_GRAD : str = "slope at x";
+pub const SAY_BATCH : str = "spec batch 0 1 2";
+pub const SAY_STATE : str = "here";
+pub const SAY_SATURATED : str = "saturated";
+pub const SAY_LEARNING : str = "still learning";
+pub const SAY_NOTE : str = "Unlike the sigmoid, tanh is centred on 0 and its slope reaches 1. It still saturates: the gradient that flows back through it is multiplied by that slope.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/activation/tanh_activation.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_band_threshold_is_a_tenth {
+ assert K_SAT_C == 10;
+ assert K_SAT_C < 100;
+}
+
+test the_presets_cover_both_regions {
+ assert K_PRESET_X_C[0] > 150;
+ assert K_PRESET_X_C[1] == 0;
+ assert K_PRESET_X_C[3] + 150 < 0;
+}
diff --git a/apps/website/specs/widgets/nn-xor.t27 b/apps/website/specs/widgets/nn-xor.t27
new file mode 100644
index 0000000000..262c463c48
--- /dev/null
+++ b/apps/website/specs/widgets/nn-xor.t27
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+; specs/widgets/nn-xor.t27 -- a 2-8-1 network learns XOR by backpropagation
+; Source of truth for public/widgets/nn-xor/ (gHashTag/trinity): scripts/widget-pages-from-spec.mjs
+; compiles this file, checks its constants, evaluates the tests below and writes the page; the page
+; hands every constant to tool.js as window.T27_WIDGET. tool.js holds no English and NO FORMULA of
+; its own: every value it draws comes from public/widgets/nn-xor/logic.js -- the fn bodies of
+; gHashTag/t27 specs/ml/pathway/mlp.t27 compiled to wasm, beside its copy logic.t27.
+; Lesson 20 of the course neural-networks. ASCII only (L3), English only (LANG-EN).
+; Numbers are integers (the schema reads no float): a K_ name ending in _C is in hundredths.
+
+; WHAT IT DOES: the reader drags the epochs from 0 to 2000. Every output is xor(epochs, 2 + c), the
+; loss curve xor(e, 1) every 25 epochs and the first-epoch loss xor(e, 0): the spec trains its own
+; 2-8-1 network from fixed initial weights at lr 0.05, from scratch on every call.
+
+; WHAT IT DOES NOT CLAIM: the grid shows the four inputs the spec exposes, not a decision region
+; over the plane: the spec has no fn for a point between the corners.
+; phi^2 + 1/phi^2 = 3 | TRINITY
+
+module widget_nn_xor;
+
+pub const KIND : str = "widget-tool";
+pub const ID : str = "nn-xor";
+pub const TITLE : str = "XOR: a network learns what one neuron cannot";
+pub const DESCRIPTION : str = "A 2-8-1 network trained by backpropagation in your browser: drag the epochs from 0 to 2000 and watch its four answers move to 0, 1, 1, 0 while the loss falls from 0.89 by twelve orders.";
+pub const IMAGE_ALT : str = "XOR card after 2000 epochs: a 2 x 2 grid of the four cases, dark blue 0.000 on the diagonal and white 1.000 off it. Beside it a red loss curve on a log axis falls from 0.89 at epoch 1 to about 1e-13 at 2000. Tiles read first-epoch loss 0.8896, last 1.78e-13, outputs 0.00 1.00 1.00 0.00 against the targets 0 1 1 0.";
+pub const HOOK : str = "One neuron cannot draw XOR. Eight hidden units can: after 2000 epochs the outputs are 0, 1, 1, 0 to within 4e-7 and the loss is 1.8e-13.";
+pub const CATEGORY : str = "compiler";
+pub const DATA_SOURCES : [2]str = ["gHashTag/t27 specs/ml/pathway/mlp.t27 (t27 master 239b4b059): every value on the page is its fn bodies, compiled to wasm (logic.js)", "Course 10 of t27.ai, neural-networks (gHashTag/trinity#1695): lesson 20 opens this widget"];
+pub const READS_LOCAL_FILES : bool = false;
+pub const SENDS_NOTHING : bool = true;
+
+pub const K_CARD_WIDTH : u16 = 1200;
+pub const K_CARD_HEIGHT : u16 = 630;
+
+; --- Numbers: ranges, presets, and fixtures the spec holds (held equal by the tests below)
+pub const K_EPOCHS_HI : u16 = 2000;
+pub const K_CURVE_STEP : u8 = 25;
+pub const K_LOG_LO : i8 = -14;
+pub const K_LOG_HI : u8 = 1;
+pub const K_LOG_TICKS : [5]i8 = [-12, -9, -6, -3, 0];
+pub const K_EPOCH_TICKS : [5]u16 = [0, 500, 1000, 1500, 2000];
+pub const K_CASES : [4]u8 = [0, 1, 1, 0];
+pub const K_PRESET_EPOCHS : [4]u16 = [2000, 0, 100, 500];
+pub const K_START_PRESET : u8 = 0;
+
+; --- Words on screen ------------------------------------------------------------------------
+pub const SAY_PRESETS_LABEL : str = "Start from";
+pub const SAY_PRESET_NAMES : [4]str = ["Trained, 2000 epochs", "Untrained", "100 epochs", "500 epochs"];
+pub const SAY_ASK : str = "Can a network learn something one neuron cannot?";
+pub const SAY_EPOCHS : str = "epochs";
+pub const SAY_A : str = "a =";
+pub const SAY_B : str = "b =";
+pub const SAY_CURVE_LABEL : str = "The training loss, log scale";
+pub const SAY_GRID_LABEL : str = "The network's answer for each input (a, b)";
+pub const SAY_LOG_NOTE : str = "log scale";
+pub const SAY_FIRST_LOSS : str = "loss, epoch 1";
+pub const SAY_LAST_LOSS : str = "loss, last epoch";
+pub const SAY_UNTRAINED : str = "not trained";
+pub const SAY_OUTPUTS : str = "outputs 00 01 10 11";
+pub const SAY_TARGETS : str = "targets";
+pub const SAY_NOTE : str = "XOR is 1 when exactly one input is 1. No straight line separates its 1s from its 0s, so one neuron fails; a hidden layer of ReLUs bends the space until one does.";
+pub const SAY_SOURCE : str = "Logic: t27 spec ml/pathway/mlp.t27, compiled";
+
+test the_card_is_the_share_size {
+ assert K_CARD_WIDTH == 1200;
+ assert K_CARD_HEIGHT == 630;
+ assert K_CARD_WIDTH * 21 == K_CARD_HEIGHT * 40;
+}
+
+test the_curve_steps_divide_the_run {
+ assert K_EPOCHS_HI % K_CURVE_STEP == 0;
+ assert K_EPOCHS_HI / K_CURVE_STEP == 80;
+}
+
+test the_targets_are_xor {
+ assert K_CASES[0] == 0;
+ assert K_CASES[1] == 1;
+ assert K_CASES[2] == 1;
+ assert K_CASES[3] == 0;
+ assert K_PRESET_EPOCHS[0] == K_EPOCHS_HI;
+}
diff --git a/apps/website/src/components/Navigation.tsx b/apps/website/src/components/Navigation.tsx
index 6ebbdb3bd7..e69a7da76b 100644
--- a/apps/website/src/components/Navigation.tsx
+++ b/apps/website/src/components/Navigation.tsx
@@ -54,6 +54,7 @@ const PAGES: PageLink[] = [
{ href: '#/buses-and-peripherals', en: 'Course: buses and peripherals', ru: 'Курс: шины и периферия', note: 'UART, SPI, APB, AXI4, Ethernet, 27 lessons', noteRu: 'UART, SPI, APB, AXI4, Ethernet, 27 уроков' },
{ href: '#/ternary-computing', en: 'Course: the ternary machine', ru: 'Курс: троичная машина', note: 'Trits, logic, adders, a machine, neurons; a live widget per lesson, 27 lessons', noteRu: 'Триты, логика, сумматоры, машина, нейроны; живой виджет в каждом уроке, 27 уроков' },
{ href: '#/arithmetic-and-dsp', en: 'Course: arithmetic and DSP', ru: 'Курс: арифметика и DSP', note: 'Adders, multipliers, fixed point, CORDIC, FIR, CIC, FFT, 27 lessons', noteRu: 'Сумматоры, умножители, фиксированная точка, CORDIC, FIR, CIC, БПФ, 27 уроков' },
+ { href: '#/neural-networks', en: 'Course: neural networks', ru: 'Курс: нейросети', note: 'Activations, losses, Adam, backprop, XOR, attention, a transformer block, 27 lessons', noteRu: 'Активации, потери, Adam, обратное распространение, XOR, внимание, блок трансформера, 27 уроков' },
{ href: '#/foundry', en: 'Golden Foundry', ru: 'Золотая Литейная', note: 'A club for people who build on silicon', noteRu: 'Клуб разработчиков на кремнии', color: '#C9A24B' },
{ href: '#/about', en: 'About', ru: 'Об авторе', note: 'Background, papers, contact', noteRu: 'Биография, статьи, контакты' },
{ href: '#/resources', en: 'Resources', ru: 'Материалы', note: 'Papers and datasets, each with a DOI', noteRu: 'Статьи и датасеты, у каждого DOI' },
diff --git a/apps/website/src/data/blog/bodies/neural-networks-in-t27-a-widget-per-lesson.ts b/apps/website/src/data/blog/bodies/neural-networks-in-t27-a-widget-per-lesson.ts
new file mode 100644
index 0000000000..86e1ea416c
--- /dev/null
+++ b/apps/website/src/data/blog/bodies/neural-networks-in-t27-a-widget-per-lesson.ts
@@ -0,0 +1,235 @@
+import type { Block } from '../types'
+
+// Course 10, neural-networks: 27 widgets, each running the fn bodies of a gHashTag/t27 specs/ml spec
+// compiled to wasm (999 scripts/t27-logic.mjs, then scripts/widget-logic.mjs). The figure was drawn
+// from the compiled mlp.t27 the nn-xor widget runs: xor(e, 1) every 25 epochs and xor(2000, 2..5).
+
+export const body: Block[] = [
+ {
+ "kind": "p",
+ "text": "Course 10 of t27.ai is out: Neural networks in t27, from one neuron to a transformer block. The agents of this project think with neural networks, and the parts those networks are made of already live in gHashTag/t27 as specs with numeric tests: activations, layers, losses, optimizers, recurrent cells, attention, a transformer block. The course opens them one by one. It has 27 lessons in 9 modules, each lesson opens with a question a curious person would ask, and its widget answers it with numbers that are the spec's own fn bodies, compiled to WebAssembly and run in your browser."
+ },
+ {
+ "kind": "h",
+ "text": "Nine modules, twenty-seven widgets"
+ },
+ {
+ "kind": "table",
+ "head": [
+ "Module",
+ "Widgets",
+ "Specs in specs/ml"
+ ],
+ "rows": [
+ [
+ "One neuron",
+ "nn-relu, nn-sigmoid, nn-tanh",
+ "relu_activation, sigmoid_activation, tanh_activation"
+ ],
+ [
+ "New bends",
+ "nn-gelu, nn-silu, nn-elu",
+ "gelu_activation, silu_swish_activation, elu_activation"
+ ],
+ [
+ "A layer",
+ "nn-softmax, nn-embedding, nn-dropout",
+ "softmax, embedding_layer, dropout_layer"
+ ],
+ [
+ "Pictures",
+ "nn-conv, nn-maxpool, nn-batchnorm",
+ "avgpool2d_layer, maxpool2d_layer, batchnorm_layer"
+ ],
+ [
+ "The error",
+ "nn-mse, nn-bce, nn-huber",
+ "mse_loss, binary_crossentropy_loss, huber_loss"
+ ],
+ [
+ "Downhill",
+ "nn-sgd, nn-rmsprop, nn-adam",
+ "sgd, rmsprop, adam"
+ ],
+ [
+ "Learning",
+ "nn-backprop, nn-xor, nn-lamb",
+ "dense_layer, mlp, lamb"
+ ],
+ [
+ "Memory and attention",
+ "nn-rnn, nn-lstm, nn-attention",
+ "rnn_cell, lstm_single, attention_mechanism"
+ ],
+ [
+ "The transformer and the reward",
+ "nn-positions, nn-encoder, nn-ppo",
+ "positional_encoding, encoder_block, ppo_clip_loss"
+ ]
+ ]
+ },
+ {
+ "kind": "p",
+ "text": "The pictures are what the numbers look like: activation curves you drag along with their slopes, softmax bars that sharpen as you cool the temperature, pooling windows as heat-maps, a weight walking down a bowl under SGD, RMSProp and Adam, two heads of attention as grids, and positions as the hands of four clocks."
+ },
+ {
+ "kind": "h",
+ "text": "A network that learns in your browser"
+ },
+ {
+ "kind": "p",
+ "text": "Lesson 20 trains a network. mlp.t27 holds a 2-8-1 network of ReLUs and a test that it learns XOR in 2000 epochs at lr 0.05. Compiled to wasm, the same xor(epochs, which) fn runs in the widget, from scratch for every point of its loss curve. The loss falls from 0.8896 in epoch 1 to 1.8e-13, and the four answers come out within 4e-7 of 0, 1, 1 and 0. The figure below is drawn by that compiled fn."
+ },
+ {
+ "kind": "figure",
+ "svg": "",
+ "caption": "The XOR training loss of the 2-8-1 network in mlp.t27, per epoch on a log scale, and its four outputs after 2000 epochs, computed by the compiled spec the nn-xor widget runs. The loss falls from 0.8896 to 1.8e-13 and flattens near the precision of f32 after about 600 epochs."
+ },
+ {
+ "kind": "h",
+ "text": "How the specs got into the browser"
+ },
+ {
+ "kind": "p",
+ "text": "No widget copies a formula. scripts/t27-logic.mjs of 999-multibots-telegraf runs t27c gen on a spec, builds the Zig with Zig 0.16 for wasm32 and writes a small module with one function per spec fn; scripts/widget-logic.mjs of the site puts it beside the widget with a byte-identical copy of the spec and stops the build when the two drift apart. The specs/ml fns take an index as usize, which t27-logic.mjs learned to lower in 999#4017; after that, 27 specs gave the 27 widgets their logic without a line rewritten. The site's spec reader learned to read an epsilon written as 1e-8 in the same change."
+ },
+ {
+ "kind": "h",
+ "text": "What it does not show"
+ },
+ {
+ "kind": "ul",
+ "items": [
+ "Lesson 10 is average pooling over 2 x 2 windows, not a convolution: the sliding conv2d spec, t27#10612, is still open, and the lesson says so.",
+ "The XOR grid shows the four inputs the spec exposes, not a decision region over the plane: the spec has no fn for a point between the corners.",
+ "The fixtures are the specs' own small ones, 3 inputs, 4 tokens, 3 positions: nothing here was trained on real data."
+ ]
+ },
+ {
+ "kind": "h",
+ "text": "Try it"
+ },
+ {
+ "kind": "ul",
+ "items": [
+ "Course: t27.ai/learn/neural-networks/",
+ "t27.ai/widgets/nn-softmax/, t27.ai/widgets/nn-xor/, t27.ai/widgets/nn-attention/, t27.ai/widgets/nn-positions/",
+ "Specs: gHashTag/t27 specs/ml/"
+ ]
+ }
+]
+
+export const ruBody: Block[] = [
+ {
+ "kind": "p",
+ "text": "На t27.ai вышел курс 10: Нейросети на t27, от одного нейрона до блока трансформера. Агенты этого проекта думают нейросетями, а части, из которых эти сети собраны, уже лежат в gHashTag/t27 спеками с числовыми тестами: функции активации, слои, функции потерь, оптимизаторы, рекуррентные ячейки, внимание, блок трансформера. Курс открывает их по одной. В нём 27 уроков в 9 модулях, каждый урок начинается с вопроса, который задал бы любопытный человек, и виджет отвечает на него числами, которые вычисляют тела fn самой спеки, скомпилированные в WebAssembly и запущенные в вашем браузере."
+ },
+ {
+ "kind": "h",
+ "text": "Девять модулей, двадцать семь виджетов"
+ },
+ {
+ "kind": "table",
+ "head": [
+ "Модуль",
+ "Виджеты",
+ "Спеки в specs/ml"
+ ],
+ "rows": [
+ [
+ "Один нейрон",
+ "nn-relu, nn-sigmoid, nn-tanh",
+ "relu_activation, sigmoid_activation, tanh_activation"
+ ],
+ [
+ "Новые изгибы",
+ "nn-gelu, nn-silu, nn-elu",
+ "gelu_activation, silu_swish_activation, elu_activation"
+ ],
+ [
+ "Слой",
+ "nn-softmax, nn-embedding, nn-dropout",
+ "softmax, embedding_layer, dropout_layer"
+ ],
+ [
+ "Картинки",
+ "nn-conv, nn-maxpool, nn-batchnorm",
+ "avgpool2d_layer, maxpool2d_layer, batchnorm_layer"
+ ],
+ [
+ "Ошибка",
+ "nn-mse, nn-bce, nn-huber",
+ "mse_loss, binary_crossentropy_loss, huber_loss"
+ ],
+ [
+ "Под гору",
+ "nn-sgd, nn-rmsprop, nn-adam",
+ "sgd, rmsprop, adam"
+ ],
+ [
+ "Обучение",
+ "nn-backprop, nn-xor, nn-lamb",
+ "dense_layer, mlp, lamb"
+ ],
+ [
+ "Память и внимание",
+ "nn-rnn, nn-lstm, nn-attention",
+ "rnn_cell, lstm_single, attention_mechanism"
+ ],
+ [
+ "Трансформер и награда",
+ "nn-positions, nn-encoder, nn-ppo",
+ "positional_encoding, encoder_block, ppo_clip_loss"
+ ]
+ ]
+ },
+ {
+ "kind": "p",
+ "text": "Картинки показывают, как выглядят числа: кривые активации, по которым можно вести курсор, вместе с наклоном, столбики softmax, которые заостряются, когда охлаждаешь температуру, окна пулинга как тепловые карты, вес, спускающийся в чашу под SGD, RMSProp и Adam, две головы внимания как сетки и позиции как стрелки четырёх часов."
+ },
+ {
+ "kind": "h",
+ "text": "Сеть, которая учится в браузере"
+ },
+ {
+ "kind": "p",
+ "text": "Урок 20 обучает сеть. mlp.t27 хранит сеть 2-8-1 из ReLU и тест, что она выучивает XOR за 2000 эпох при lr 0.05. Собранная в wasm, та же fn xor(epochs, which) работает в виджете, заново для каждой точки кривой потерь. Потеря падает с 0.8896 в эпохе 1 до 1.8e-13, а четыре ответа выходят в пределах 4e-7 от 0, 1, 1 и 0. Рисунок ниже нарисован этой собранной fn."
+ },
+ {
+ "kind": "figure",
+ "svg": "",
+ "caption": "Потеря обучения XOR сети 2-8-1 из mlp.t27 по эпохам в логарифмической шкале и четыре её выхода после 2000 эпох, посчитанные собранной спекой, которую исполняет виджет nn-xor. Потеря падает с 0.8896 до 1.8e-13 и выполаживается около точности f32 примерно после 600 эпох."
+ },
+ {
+ "kind": "h",
+ "text": "Как спеки попали в браузер"
+ },
+ {
+ "kind": "p",
+ "text": "Ни один виджет не копирует формулу. scripts/t27-logic.mjs из 999-multibots-telegraf запускает t27c gen на спеке, собирает Zig через Zig 0.16 под wasm32 и пишет небольшой модуль с одной функцией на каждую fn спеки; scripts/widget-logic.mjs сайта кладёт его рядом с виджетом вместе с побайтной копией спеки и останавливает сборку, если они разошлись. Функции specs/ml принимают индекс как usize, и t27-logic.mjs научился его понижать в 999#4017; после этого 27 спек дали 27 виджетам их логику без единой переписанной строки. Читатель спек на сайте в том же изменении научился читать эпсилон, записанный как 1e-8."
+ },
+ {
+ "kind": "h",
+ "text": "Чего это не показывает"
+ },
+ {
+ "kind": "ul",
+ "items": [
+ "Урок 10 это усредняющий пулинг по окнам 2 x 2, а не свёртка: спека скользящей conv2d, t27#10612, ещё открыта, и урок говорит об этом.",
+ "Сетка XOR показывает четыре входа, которые открывает спека, а не область решения на плоскости: у спеки нет fn для точки между углами.",
+ "Фикстуры это маленькие фикстуры самих спек, 3 входа, 4 токена, 3 позиции: ничто здесь не обучалось на настоящих данных."
+ ]
+ },
+ {
+ "kind": "h",
+ "text": "Попробовать"
+ },
+ {
+ "kind": "ul",
+ "items": [
+ "Курс: t27.ai/learn/neural-networks/",
+ "t27.ai/widgets/nn-softmax/, t27.ai/widgets/nn-xor/, t27.ai/widgets/nn-attention/, t27.ai/widgets/nn-positions/",
+ "Спеки: gHashTag/t27 specs/ml/"
+ ]
+ }
+]
diff --git a/apps/website/src/data/blog/index.ts b/apps/website/src/data/blog/index.ts
index 77f37996f0..0665822df9 100644
--- a/apps/website/src/data/blog/index.ts
+++ b/apps/website/src/data/blog/index.ts
@@ -2,6 +2,37 @@ import type { PostMeta } from './types'
/** Индекс блога: список и метаданные без тяжёлых тел публикаций. */
export const postsIndex: PostMeta[] = [
+ {
+ slug: "neural-networks-in-t27-a-widget-per-lesson",
+ title: "Neural networks in t27: 27 lessons, and a widget in each one that runs a specs/ml spec",
+ summary: "[every widget runs fn bodies of a gHashTag/t27 specs/ml spec compiled to wasm; lesson 10 is average pooling until t27#10612 merges; the fixtures are the specs' own, nothing is trained on real data] Course 10 opens the parts the project's agents think with: ReLU, GELU and softmax, embeddings and dropout, pooling and batch norm, MSE, cross-entropy and Huber, SGD, RMSProp, Adam and LAMB, backpropagation, a 2-8-1 network that learns XOR in the browser, a recurrent cell, LSTM gates, attention, positional encoding, an encoder block and PPO.",
+ date: "2026-10-18",
+ readingMinutes: 4,
+ tags: ["t27", "Course", "AI"],
+ receipts: [
+ { label: "trinity#1695: the course issue, 27 widgets on gHashTag/t27 specs/ml", href: "https://github.com/gHashTag/trinity/issues/1695" },
+ { label: "The course: Neural networks in t27", href: "https://t27.ai/learn/neural-networks/" },
+ { label: "The XOR widget: a network trained in the browser", href: "https://t27.ai/widgets/nn-xor/" },
+ { label: "The softmax widget", href: "https://t27.ai/widgets/nn-softmax/" },
+ { label: "The attention widget", href: "https://t27.ai/widgets/nn-attention/" },
+ { label: "t27#10612: the sliding conv2d spec lesson 10 waits for", href: "https://github.com/gHashTag/t27/issues/10612" },
+ ],
+ openQuestions: [
+ "Lesson 10 averages each 2 x 2 window; a convolution with a learned kernel waits for t27#10612.",
+ "The XOR widget shows the four inputs the spec exposes, not a decision region over the plane.",
+ "Every fixture is a spec's own small one; no widget runs a model trained on real data.",
+ ],
+ published: true,
+ ru: {
+ title: "Нейросети на t27: 27 уроков, и в каждом виджет, который исполняет спеку specs/ml",
+ summary: "[каждый виджет исполняет тела функций спеки gHashTag/t27 specs/ml, собранные в wasm; урок 10 это усредняющий пулинг, пока не слита t27#10612; фикстуры взяты из самих спек, на настоящих данных ничего не обучалось] Курс 10 открывает части, которыми думают агенты проекта: ReLU, GELU и softmax, эмбеддинги и dropout, пулинг и пакетную нормализацию, MSE, перекрёстную энтропию и функцию Хьюбера, SGD, RMSProp, Adam и LAMB, обратное распространение, сеть 2-8-1, которая учит XOR в браузере, рекуррентную ячейку, вентили LSTM, внимание, позиционное кодирование, блок энкодера и PPO.",
+ openQuestions: [
+ "Урок 10 усредняет каждое окно 2 x 2; свёртка с выученным ядром ждёт t27#10612.",
+ "Виджет XOR показывает четыре входа, которые открывает спека, а не область решения на плоскости.",
+ "Каждая фикстура это маленькая фикстура самой спеки; ни один виджет не исполняет модель, обученную на настоящих данных.",
+ ],
+ },
+ },
{
slug: "a-wasm-interpreter-ran-on-the-board",
title: "A WASM interpreter ran on the board: 134 of 134, from a spec",
diff --git a/apps/website/src/data/blog/posts.ts b/apps/website/src/data/blog/posts.ts
index f0cb43ab98..80b3fe53d0 100644
--- a/apps/website/src/data/blog/posts.ts
+++ b/apps/website/src/data/blog/posts.ts
@@ -1,4 +1,5 @@
import { body as body_a_wasm_interpreter_ran_on_the_board, ruBody as ruBody_a_wasm_interpreter_ran_on_the_board } from './bodies/a-wasm-interpreter-ran-on-the-board'
+import { body as body_neural_networks_in_t27_a_widget_per_lesson, ruBody as ruBody_neural_networks_in_t27_a_widget_per_lesson } from './bodies/neural-networks-in-t27-a-widget-per-lesson'
import { body as body_arithmetic_and_dsp_a_widget_per_lesson, ruBody as ruBody_arithmetic_and_dsp_a_widget_per_lesson } from './bodies/arithmetic-and-dsp-a-widget-per-lesson'
import { body as body_the_ternary_machine_course, ruBody as ruBody_the_ternary_machine_course } from './bodies/the-ternary-machine-course'
import { body as body_three_widgets_under_the_bus, ruBody as ruBody_three_widgets_under_the_bus } from './bodies/three-widgets-under-the-bus'
@@ -140,6 +141,7 @@ const bodies: Record = {
'bypass-or-a-broken-shift': { body: body_bypass_or_a_broken_shift, ruBody: ruBody_bypass_or_a_broken_shift },
'the-ternary-machine-course': { body: body_the_ternary_machine_course, ruBody: ruBody_the_ternary_machine_course },
'three-widgets-under-the-bus': { body: body_three_widgets_under_the_bus, ruBody: ruBody_three_widgets_under_the_bus },
+ 'neural-networks-in-t27-a-widget-per-lesson': { body: body_neural_networks_in_t27_a_widget_per_lesson, ruBody: ruBody_neural_networks_in_t27_a_widget_per_lesson },
'arithmetic-and-dsp-a-widget-per-lesson': { body: body_arithmetic_and_dsp_a_widget_per_lesson, ruBody: ruBody_arithmetic_and_dsp_a_widget_per_lesson },
'the-back-half-of-openxc7-from-a-spec': { body: body_the_back_half_of_openxc7_from_a_spec, ruBody: ruBody_the_back_half_of_openxc7_from_a_spec },
'tri-mined-not-sold': { body: body_tri_mined_not_sold, ruBody: ruBody_tri_mined_not_sold },
diff --git a/apps/website/src/lib/course.generated.ts b/apps/website/src/lib/course.generated.ts
index 3c3fb75ca8..858a1c0ec0 100644
--- a/apps/website/src/lib/course.generated.ts
+++ b/apps/website/src/lib/course.generated.ts
@@ -1,5 +1,5 @@
-// GENERATED by scripts/course-from-spec.mjs from specs/course/courses.t27, specs/course/t27-basics.t27, specs/course/course.t27, specs/course/ai-numbers.t27, specs/course/goldenfloat.t27, specs/course/verify-hardware.t27, specs/course/clocks-and-cdc.t27, specs/course/buses-and-peripherals.t27, specs/course/ternary-computing.t27, specs/course/arithmetic-and-dsp.t27 and their Russian bundles
-// spec sha256 t27-basics 4827f6b9f5d95ae98e1d49f037966d2cc826e07c691044f2f5fa1ee9c98ae474, course 8429472829c5d9a0782b2d2b21ff26879a429c5470f4abb5088f1e1a7fa674be, ai-numbers 7f736e357ef13446dfa8f978482cc28c7710caac3f7396769a8fc01b91e20802, goldenfloat d6b6019b8e6bf77a41991b32a3a3d1b5eb78ca4b7078e2bf5f5436df3dd7c59c, verify-hardware 7ca437575eaa8f8646f42c82b75862f091fb509708de152c4e18ed76ee39d4cf, clocks-and-cdc 7d491967f9326fcde194ab19a9bfa7f0ed960ba94d7ca3d82df0747e269c5acc, buses-and-peripherals 4dfdbd8c909d7ad8f1ac2d4f7025c77f0619ade92b181212ea9b589262695acc, ternary-computing e0a14010163f00208d5b54c800e6ce3f05e6b8581de94c0323097b00b1854437, arithmetic-and-dsp c2b6bfd642e88caadaff9da965ac49172b389427b574b27d4c64145851bc206d
+// GENERATED by scripts/course-from-spec.mjs from specs/course/courses.t27, specs/course/t27-basics.t27, specs/course/course.t27, specs/course/ai-numbers.t27, specs/course/goldenfloat.t27, specs/course/verify-hardware.t27, specs/course/clocks-and-cdc.t27, specs/course/buses-and-peripherals.t27, specs/course/ternary-computing.t27, specs/course/arithmetic-and-dsp.t27, specs/course/neural-networks.t27 and their Russian bundles
+// spec sha256 t27-basics 4827f6b9f5d95ae98e1d49f037966d2cc826e07c691044f2f5fa1ee9c98ae474, course 8429472829c5d9a0782b2d2b21ff26879a429c5470f4abb5088f1e1a7fa674be, ai-numbers 7f736e357ef13446dfa8f978482cc28c7710caac3f7396769a8fc01b91e20802, goldenfloat d6b6019b8e6bf77a41991b32a3a3d1b5eb78ca4b7078e2bf5f5436df3dd7c59c, verify-hardware 7ca437575eaa8f8646f42c82b75862f091fb509708de152c4e18ed76ee39d4cf, clocks-and-cdc 7d491967f9326fcde194ab19a9bfa7f0ed960ba94d7ca3d82df0747e269c5acc, buses-and-peripherals 4dfdbd8c909d7ad8f1ac2d4f7025c77f0619ade92b181212ea9b589262695acc, ternary-computing e0a14010163f00208d5b54c800e6ce3f05e6b8581de94c0323097b00b1854437, arithmetic-and-dsp c2b6bfd642e88caadaff9da965ac49172b389427b574b27d4c64145851bc206d, neural-networks d6cffe6af3c2bd2ce3f73272eee7cca3e463f8f8e4e034d1dbc39f7f43083bc1
// Do not edit: change the .t27 source (or a Russian bundle) and regenerate.
export const COURSES = [
@@ -11865,6 +11865,1761 @@ export const COURSES = [
"ru": "Троичная машина: от трита до работающей программы"
}
},
+ "next": {
+ "id": "neural-networks",
+ "route": "neural-networks",
+ "share": "learn/neural-networks/",
+ "first": "relu",
+ "last": "ppo-and-the-reward",
+ "title": {
+ "en": "Neural networks in t27: from one neuron to a transformer block",
+ "ru": "Нейросети на t27: от одного нейрона до блока трансформера"
+ }
+ }
+ },
+ {
+ "id": "neural-networks",
+ "source": {
+ "spec": "specs/course/neural-networks.t27",
+ "publicSpec": "public/learn/neural-networks.t27",
+ "ruContract": "specs/course/neural-networks-ru.t27",
+ "ruBundle": "apps/website/i18n/neural-networks.ru.json",
+ "sha256": "d6cffe6af3c2bd2ce3f73272eee7cca3e463f8f8e4e034d1dbc39f7f43083bc1",
+ "schemaVersion": 1,
+ "gallery": "specs/widgets/gallery.t27"
+ },
+ "route": "neural-networks",
+ "share": "learn/neural-networks/",
+ "cohortRoute": "fpga-training",
+ "progressKey": "t27-course-done",
+ "sendsNothing": true,
+ "locales": [
+ "en",
+ "ru"
+ ],
+ "say": {
+ "en": {
+ "TITLE": "Neural networks in t27: from one neuron to a transformer block",
+ "DESCRIPTION": "Course 10, 27 lessons, a live widget and a t27 spec each: activations, softmax and embeddings, pooling and batch norm, losses, SGD, RMSProp and Adam, backpropagation and XOR, recurrent cells, attention, a transformer block and PPO. Every number is computed by a compiled spec.",
+ "ALL": "All lessons",
+ "ALSO": "Also try",
+ "ALSO_HINT": "Each one swaps the widget above.",
+ "BACK_TO_MAIN": "Back to this lesson's widget",
+ "COHORT": "Looking for the paid FPGA training? It moved to its own page.",
+ "CONTINUE": "Continue",
+ "GOAL": "You will learn",
+ "KICKER": "Course",
+ "LEAD": "The agents of this project think with neural networks, and here their parts are written as t27 specs with tests, specs the compiler checks. Every widget runs the fn bodies of a specs/ml spec compiled to WebAssembly: move a slider and the spec answers.",
+ "LESSON": "Lesson {0} of {1}",
+ "MARK": "Mark as done",
+ "MARKED": "Done",
+ "MODULE": "Module {0}",
+ "NEXT": "Next",
+ "NEXT_COURSE": "Next course",
+ "NOT_FOUND": "No lesson has this address.",
+ "OPEN_COURSE": "Open the interactive course",
+ "OPEN_LESSON": "Open the interactive lesson",
+ "OPEN_PAGE": "Open the widget on its own page",
+ "PREV": "Previous",
+ "PREV_COURSE": "Previous course",
+ "PRIVATE": "Progress stays in this browser; nothing is sent.",
+ "PROGRESS": "{0} of {1} done",
+ "SEO_TITLE": "{0}: neural networks course, lesson {1} of {2}",
+ "SHAPE": "9 modules of 3 lessons, 27 cells. A filled cell is a lesson you marked done.",
+ "SHARE": "Link to share, with a preview card",
+ "SOURCE": "This course is itself a t27 spec: read it",
+ "SPEC": "Open the lesson's spec in the player",
+ "START": "Start lesson 1",
+ "TRY": "Try it",
+ "WIDGET_LANG": "Widgets keep their own English words: each one lives in its own t27 spec."
+ },
+ "ru": {
+ "TITLE": "Нейросети на t27: от одного нейрона до блока трансформера",
+ "DESCRIPTION": "Курс 10, 27 уроков, в каждом живой виджет и спека t27: функции активации, softmax и эмбеддинги, пулинг и пакетная нормализация, функции потерь, SGD, RMSProp и Adam, обратное распространение и XOR, рекуррентные ячейки, внимание, блок трансформера и PPO. Каждое число вычисляет скомпилированная спека.",
+ "ALL": "Все уроки",
+ "ALSO": "Также откройте",
+ "ALSO_HINT": "Каждая кнопка меняет виджет сверху.",
+ "BACK_TO_MAIN": "Вернуться к виджету урока",
+ "COHORT": "Ищете платное обучение по FPGA? Оно переехало на отдельную страницу.",
+ "CONTINUE": "Продолжить",
+ "GOAL": "Вы узнаете",
+ "KICKER": "Курс",
+ "LEAD": "Агенты этого проекта думают нейросетями, а здесь их части записаны спеками t27 с тестами, которые проверяет компилятор. Каждый виджет исполняет тела fn спеки из specs/ml, скомпилированные в WebAssembly: двигаете ползунок — отвечает спека.",
+ "LESSON": "Урок {0} из {1}",
+ "MARK": "Отметить пройденным",
+ "MARKED": "Пройдено",
+ "MODULE": "Модуль {0}",
+ "NEXT": "Далее",
+ "NEXT_COURSE": "Следующий курс",
+ "NOT_FOUND": "У урока нет такого адреса.",
+ "OPEN_COURSE": "Открыть интерактивный курс",
+ "OPEN_LESSON": "Открыть интерактивный урок",
+ "OPEN_PAGE": "Открыть виджет на отдельной странице",
+ "PREV": "Назад",
+ "PREV_COURSE": "Предыдущий курс",
+ "PRIVATE": "Прогресс хранится только в этом браузере; ничего не отправляется.",
+ "PROGRESS": "{0} из {1} пройдено",
+ "SEO_TITLE": "{0}: курс о нейросетях, урок {1} из {2}",
+ "SHAPE": "9 модулей по 3 урока, 27 клеток. Заполненная клетка — урок, отмеченный пройденным.",
+ "SHARE": "Ссылка для шаринга, с карточкой предпросмотра",
+ "SOURCE": "Этот курс сам является спекой t27: прочитайте её",
+ "SPEC": "Открыть спеку урока в плеере",
+ "START": "Начать урок 1",
+ "TRY": "Попробовать",
+ "WIDGET_LANG": "Виджеты хранят свои английские слова: у каждого своя спека t27."
+ }
+ },
+ "modules": [
+ {
+ "id": "one-neuron",
+ "n": 1,
+ "lessons": [
+ "relu",
+ "the-sigmoid",
+ "tanh-and-saturation"
+ ],
+ "en": {
+ "title": "One neuron",
+ "line": "A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back."
+ },
+ "ru": {
+ "title": "Один нейрон",
+ "line": "Нейрон складывает входы и изгибает сумму: ReLU, сигмоида и tanh, и где каждая перестаёт пропускать градиент назад."
+ }
+ },
+ {
+ "id": "new-bends",
+ "n": 2,
+ "lessons": [
+ "gelu",
+ "swish-and-silu",
+ "elu"
+ ],
+ "en": {
+ "title": "New bends",
+ "line": "The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff."
+ },
+ "ru": {
+ "title": "Новые изгибы",
+ "line": "Гладкие кривые нынешних моделей: GELU с провалом, Swish с ручкой и ELU с полом вместо обрыва."
+ }
+ },
+ {
+ "id": "a-layer",
+ "n": 3,
+ "lessons": [
+ "softmax-and-temperature",
+ "embeddings",
+ "dropout"
+ ],
+ "en": {
+ "title": "A layer",
+ "line": "What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off."
+ },
+ "ru": {
+ "title": "Слой",
+ "line": "Что окружает нейроны: softmax превращает оценки в выбор, эмбеддинг превращает слово в числа, dropout выключает нейроны."
+ }
+ },
+ {
+ "id": "pictures",
+ "n": 4,
+ "lessons": [
+ "patches-of-a-picture",
+ "max-pooling",
+ "batch-normalisation"
+ ],
+ "en": {
+ "title": "Pictures",
+ "line": "Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales."
+ },
+ "ru": {
+ "title": "Картинки",
+ "line": "Чтение картинки по участкам: окно, усреднённое по участку, max pooling и пакетная нормализация признаков разного масштаба."
+ }
+ },
+ {
+ "id": "the-error",
+ "n": 5,
+ "lessons": [
+ "mean-squared-error",
+ "cross-entropy",
+ "the-huber-loss"
+ ],
+ "en": {
+ "title": "The error",
+ "line": "How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers."
+ },
+ "ru": {
+ "title": "Ошибка",
+ "line": "Насколько неверна догадка, одним числом: среднеквадратичная ошибка, перекрёстная энтропия для вероятностей и функция Хьюбера для выбросов."
+ }
+ },
+ {
+ "id": "downhill",
+ "n": 6,
+ "lessons": [
+ "gradient-descent",
+ "rmsprop",
+ "adam"
+ ],
+ "en": {
+ "title": "Downhill",
+ "line": "Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them."
+ },
+ "ru": {
+ "title": "Под гору",
+ "line": "Спуск по функции потерь: градиентный спуск с моментом, RMSProp без оглядки на величину градиента и Adam, который соединяет оба."
+ }
+ },
+ {
+ "id": "learning",
+ "n": 7,
+ "lessons": [
+ "backpropagation",
+ "learning-xor",
+ "lamb-and-large-batches"
+ ],
+ "en": {
+ "title": "Learning",
+ "line": "Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches."
+ },
+ "ru": {
+ "title": "Обучение",
+ "line": "Откуда берутся градиенты и чего они добиваются: обратное распространение через слой, сеть, которая учит XOR, и LAMB для больших пакетов."
+ }
+ },
+ {
+ "id": "memory-and-attention",
+ "n": 8,
+ "lessons": [
+ "a-recurrent-cell",
+ "lstm-gates",
+ "attention"
+ ],
+ "en": {
+ "title": "Memory and attention",
+ "line": "Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once."
+ },
+ "ru": {
+ "title": "Память и внимание",
+ "line": "Сети, которые читают последовательность: рекуррентная ячейка и её затухающее эхо, вентили LSTM и внимание ко всем позициям сразу."
+ }
+ },
+ {
+ "id": "transformer-and-reward",
+ "n": 9,
+ "lessons": [
+ "positional-encoding",
+ "an-encoder-block",
+ "ppo-and-the-reward"
+ ],
+ "en": {
+ "title": "The transformer and the reward",
+ "line": "The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching."
+ },
+ "ru": {
+ "title": "Трансформер и награда",
+ "line": "Части трансформера и агента: позиции как часы, один блок энкодера и PPO, который учится на награде без рывков."
+ }
+ }
+ ],
+ "lessons": [
+ {
+ "id": "relu",
+ "n": 1,
+ "module": "one-neuron",
+ "widget": {
+ "id": "nn-relu",
+ "kind": "tool",
+ "title": "ReLU: what a neuron does with a negative number",
+ "url": "https://t27.ai/widgets/nn-relu/",
+ "preview": "widgets/nn-relu/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-sigmoid",
+ "kind": "tool",
+ "title": "Sigmoid: turn any number into a probability",
+ "url": "https://t27.ai/widgets/nn-sigmoid/",
+ "preview": "widgets/nn-sigmoid/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-tanh",
+ "kind": "tool",
+ "title": "tanh: where a neuron stops learning",
+ "url": "https://t27.ai/widgets/nn-tanh/",
+ "preview": "widgets/nn-tanh/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "tc-one-neuron",
+ "kind": "tool",
+ "title": "One ternary neuron: inputs, weights, a threshold",
+ "url": "https://t27.ai/widgets/tc-one-neuron/",
+ "preview": "widgets/tc-one-neuron/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/activation/relu_activation.t27",
+ "source": "t27/files/specs/ml/activation/relu_activation.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Factivation%2Frelu_activation.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "ReLU: keep the positive part",
+ "goal": "What ReLU does to a positive and to a negative input, and why a slope of 0 stops a unit from learning.",
+ "text": "A neuron adds up its weighted inputs and passes the sum through an activation. ReLU is the plainest one, max(0, x): a positive sum goes through unchanged and a negative one becomes 0. Its slope is 1 or 0, and a slope of 0 is a unit that has stopped learning, because no gradient flows back through it. A leaky slope fixes that: with a slope of 0.1, x = -2 gives -0.2 and passes back a gradient of 0.1. The spec relu_activation.t27 computes both curves the widget draws.",
+ "task": "Put x at -2 and read the slope with and without the leak; then find the slope at x = 1.5."
+ },
+ "ru": {
+ "title": "ReLU: оставить положительную часть",
+ "goal": "Что ReLU делает с положительным и с отрицательным входом и почему наклон 0 останавливает обучение нейрона.",
+ "text": "Нейрон складывает взвешенные входы и пропускает сумму через функцию активации. ReLU из них самая простая, max(0, x): положительная сумма проходит без изменений, отрицательная становится 0. Её наклон равен 1 или 0, а наклон 0 означает нейрон, который перестал учиться: градиент через него назад не течёт. Утечка это исправляет: с наклоном 0.1 вход x = -2 даёт -0.2 и пропускает назад градиент 0.1. Спека relu_activation.t27 вычисляет обе кривые, которые рисует виджет.",
+ "task": "Поставьте x на -2 и прочитайте наклон с утечкой и без неё; затем найдите наклон при x = 1.5."
+ }
+ },
+ {
+ "id": "the-sigmoid",
+ "n": 2,
+ "module": "one-neuron",
+ "widget": {
+ "id": "nn-sigmoid",
+ "kind": "tool",
+ "title": "Sigmoid: turn any number into a probability",
+ "url": "https://t27.ai/widgets/nn-sigmoid/",
+ "preview": "widgets/nn-sigmoid/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-relu",
+ "kind": "tool",
+ "title": "ReLU: what a neuron does with a negative number",
+ "url": "https://t27.ai/widgets/nn-relu/",
+ "preview": "widgets/nn-relu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-tanh",
+ "kind": "tool",
+ "title": "tanh: where a neuron stops learning",
+ "url": "https://t27.ai/widgets/nn-tanh/",
+ "preview": "widgets/nn-tanh/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-bce",
+ "kind": "tool",
+ "title": "Cross-entropy: why a confident wrong answer costs so much",
+ "url": "https://t27.ai/widgets/nn-bce/",
+ "preview": "widgets/nn-bce/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/activation/sigmoid_activation.t27",
+ "source": "t27/files/specs/ml/activation/sigmoid_activation.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Factivation%2Fsigmoid_activation.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "The sigmoid: a number into a probability",
+ "goal": "How the sigmoid maps any input to a probability, and why its slope limits deep networks.",
+ "text": "The sigmoid 1 / (1 + e^-x) squeezes the whole number line into 0 to 1, which is why it reads as a probability: 2 becomes 0.8808 and -4 becomes 0.018. Its slope is s (1 - s), computed from the output s itself, and it is largest at 0, where it is 0.25; at x = 8 it is 0.000335. A gradient that passes back through a stack of sigmoids is multiplied by at most 0.25 at each one, which is why deep sigmoid networks trained slowly. The spec sigmoid_activation.t27 computes the curve and the slope.",
+ "task": "Find the input where the probability is 0.5; then drag to 6 and read how little slope is left."
+ },
+ "ru": {
+ "title": "Сигмоида: число в вероятность",
+ "goal": "Как сигмоида переводит любой вход в вероятность и почему её наклон мешает глубоким сетям.",
+ "text": "Сигмоида 1 / (1 + e^-x) сжимает всю числовую прямую в отрезок от 0 до 1, поэтому её читают как вероятность: 2 превращается в 0.8808, а -4 в 0.018. Её наклон равен s (1 - s) и считается по самому выходу s; он наибольший в 0, где равен 0.25, а при x = 8 равен 0.000335. Градиент, который идёт назад через стопку сигмоид, на каждой умножается не больше чем на 0.25, поэтому глубокие сети на сигмоидах учились медленно. Спека sigmoid_activation.t27 вычисляет кривую и наклон.",
+ "task": "Найдите вход, при котором вероятность равна 0.5; затем перетащите к 6 и посмотрите, как мало осталось наклона."
+ }
+ },
+ {
+ "id": "tanh-and-saturation",
+ "n": 3,
+ "module": "one-neuron",
+ "widget": {
+ "id": "nn-tanh",
+ "kind": "tool",
+ "title": "tanh: where a neuron stops learning",
+ "url": "https://t27.ai/widgets/nn-tanh/",
+ "preview": "widgets/nn-tanh/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-relu",
+ "kind": "tool",
+ "title": "ReLU: what a neuron does with a negative number",
+ "url": "https://t27.ai/widgets/nn-relu/",
+ "preview": "widgets/nn-relu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-sigmoid",
+ "kind": "tool",
+ "title": "Sigmoid: turn any number into a probability",
+ "url": "https://t27.ai/widgets/nn-sigmoid/",
+ "preview": "widgets/nn-sigmoid/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/activation/tanh_activation.t27",
+ "source": "t27/files/specs/ml/activation/tanh_activation.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Factivation%2Ftanh_activation.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "tanh and saturation",
+ "goal": "Why tanh is centred on zero, and where its slope falls so low that a unit stops learning.",
+ "text": "tanh is a sigmoid stretched to -1..1 and centred on 0, so a layer of tanh units passes on values that average near zero, and its slope reaches 1 at the centre. It still saturates: at 2.5 the output is 0.9866 and the slope only 0.0266. The red band marks every input where the slope is under 0.1, beyond about 1.8 on either side, and a unit whose inputs live there barely learns. The spec tanh_activation.t27 computes the curve, the slope and its own batch, tanh of 0, 1 and 2: 0, 0.762 and 0.964.",
+ "task": "Drag x until the band starts and read the slope there; then compare the slope at 0 and at 2.5."
+ },
+ "ru": {
+ "title": "tanh и насыщение",
+ "goal": "Почему tanh центрирована на нуле и где её наклон падает так низко, что нейрон перестаёт учиться.",
+ "text": "tanh это сигмоида, растянутая на -1..1 и центрированная на 0, поэтому слой нейронов tanh передаёт дальше значения со средним около нуля, а её наклон в центре достигает 1. Она тоже насыщается: при 2.5 выход 0.9866, а наклон всего 0.0266. Красная полоса отмечает все входы, где наклон меньше 0.1, дальше примерно 1.8 в обе стороны, и нейрон, чьи входы живут там, почти не учится. Спека tanh_activation.t27 вычисляет кривую, наклон и свой пакет, tanh от 0, 1 и 2: 0, 0.762 и 0.964.",
+ "task": "Тяните x, пока не начнётся полоса, и прочитайте наклон там; затем сравните наклон в 0 и при 2.5."
+ }
+ },
+ {
+ "id": "gelu",
+ "n": 4,
+ "module": "new-bends",
+ "widget": {
+ "id": "nn-gelu",
+ "kind": "tool",
+ "title": "GELU: why transformers bend a little below zero",
+ "url": "https://t27.ai/widgets/nn-gelu/",
+ "preview": "widgets/nn-gelu/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-silu",
+ "kind": "tool",
+ "title": "Swish: one knob between a smooth curve and ReLU",
+ "url": "https://t27.ai/widgets/nn-silu/",
+ "preview": "widgets/nn-silu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-elu",
+ "kind": "tool",
+ "title": "ELU: a negative side with a floor",
+ "url": "https://t27.ai/widgets/nn-elu/",
+ "preview": "widgets/nn-elu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-encoder",
+ "kind": "tool",
+ "title": "An encoder block: what one transformer layer does",
+ "url": "https://t27.ai/widgets/nn-encoder/",
+ "preview": "widgets/nn-encoder/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/activation/gelu_activation.t27",
+ "source": "t27/files/specs/ml/activation/gelu_activation.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Factivation%2Fgelu_activation.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "GELU: a dip below zero",
+ "goal": "What GELU computes, why it dips below zero, and how close its tanh shortcut comes.",
+ "text": "GELU weighs x by the chance that a standard normal draw is below it: GELU(x) = x Phi(x). A large positive input passes, a large negative one vanishes, and in between the curve dips to about -0.17 near x = -0.75 before it rises. BERT and GPT use it in their feed-forward layers. The exact form needs erf, which the spec computes with the Abramowitz-Stegun fit, good to 1.5e-7, and it also computes the tanh shortcut: at -0.75 they give -0.16997 and -0.17004. The spec gelu_activation.t27 computes both curves and their slopes.",
+ "task": "Find the lowest point of the curve by its slope; then compare the exact and the tanh form at -3."
+ },
+ "ru": {
+ "title": "GELU: провал ниже нуля",
+ "goal": "Что вычисляет GELU, почему она проваливается ниже нуля и насколько близко подходит её приближение через tanh.",
+ "text": "GELU взвешивает x вероятностью того, что стандартная нормальная величина меньше него: GELU(x) = x Phi(x). Большой положительный вход проходит, большой отрицательный исчезает, а между ними кривая проваливается примерно до -0.17 около x = -0.75 и только потом растёт. BERT и GPT используют её в своих слоях прямого распространения. Точной форме нужна erf, и спека вычисляет её приближением Абрамовица-Стиган с точностью 1.5e-7, а также вычисляет приближение через tanh: при -0.75 они дают -0.16997 и -0.17004. Спека gelu_activation.t27 вычисляет обе кривые и их наклоны.",
+ "task": "Найдите нижнюю точку кривой по её наклону; затем сравните точную форму и форму через tanh при -3."
+ }
+ },
+ {
+ "id": "swish-and-silu",
+ "n": 5,
+ "module": "new-bends",
+ "widget": {
+ "id": "nn-silu",
+ "kind": "tool",
+ "title": "Swish: one knob between a smooth curve and ReLU",
+ "url": "https://t27.ai/widgets/nn-silu/",
+ "preview": "widgets/nn-silu/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-gelu",
+ "kind": "tool",
+ "title": "GELU: why transformers bend a little below zero",
+ "url": "https://t27.ai/widgets/nn-gelu/",
+ "preview": "widgets/nn-gelu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-elu",
+ "kind": "tool",
+ "title": "ELU: a negative side with a floor",
+ "url": "https://t27.ai/widgets/nn-elu/",
+ "preview": "widgets/nn-elu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-relu",
+ "kind": "tool",
+ "title": "ReLU: what a neuron does with a negative number",
+ "url": "https://t27.ai/widgets/nn-relu/",
+ "preview": "widgets/nn-relu/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/activation/silu_swish_activation.t27",
+ "source": "t27/files/specs/ml/activation/silu_swish_activation.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Factivation%2Fsilu_swish_activation.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Swish and SiLU",
+ "goal": "How x times sigmoid(beta x) gates itself, and how one parameter moves it from a smooth curve to ReLU.",
+ "text": "Swish is x times sigmoid(beta x), a unit that gates itself. At beta 1 it is called SiLU, and its lowest point is -0.2785 near x = -1.28. Raise beta and the gate sharpens: at beta 8 the curve is ReLU with rounded corners; at beta 0.5 it is a gentle slope through zero. One parameter moves the unit between a smooth curve and a switch, and a network can learn it. The spec silu_swish_activation.t27 computes the curve and its slope for any beta, and its own batch at -2, 0 and 2: -0.238, 0 and 1.762.",
+ "task": "Hold x at -1.28 and raise beta from 1 to 8; then find the beta at which the dip is gone at this scale."
+ },
+ "ru": {
+ "title": "Swish и SiLU",
+ "goal": "Как x, умноженный на sigmoid(beta x), открывает сам себя и как один параметр переводит его от гладкой кривой к ReLU.",
+ "text": "Swish это x, умноженный на sigmoid(beta x): нейрон, который сам себе вентиль. При beta 1 он называется SiLU, и его нижняя точка -0.2785 около x = -1.28. Увеличьте beta, и вентиль становится резче: при beta 8 кривая это ReLU со скруглёнными углами, при beta 0.5 пологий склон через ноль. Один параметр переводит нейрон от гладкой кривой к переключателю, и сеть может его выучить. Спека silu_swish_activation.t27 вычисляет кривую и её наклон при любом beta, а также свой пакет при -2, 0 и 2: -0.238, 0 и 1.762.",
+ "task": "Держите x на -1.28 и поднимайте beta от 1 до 8; затем найдите beta, при котором провал на этом масштабе исчезает."
+ }
+ },
+ {
+ "id": "elu",
+ "n": 6,
+ "module": "new-bends",
+ "widget": {
+ "id": "nn-elu",
+ "kind": "tool",
+ "title": "ELU: a negative side with a floor",
+ "url": "https://t27.ai/widgets/nn-elu/",
+ "preview": "widgets/nn-elu/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-relu",
+ "kind": "tool",
+ "title": "ReLU: what a neuron does with a negative number",
+ "url": "https://t27.ai/widgets/nn-relu/",
+ "preview": "widgets/nn-relu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-silu",
+ "kind": "tool",
+ "title": "Swish: one knob between a smooth curve and ReLU",
+ "url": "https://t27.ai/widgets/nn-silu/",
+ "preview": "widgets/nn-silu/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-gelu",
+ "kind": "tool",
+ "title": "GELU: why transformers bend a little below zero",
+ "url": "https://t27.ai/widgets/nn-gelu/",
+ "preview": "widgets/nn-gelu/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/activation/elu_activation.t27",
+ "source": "t27/files/specs/ml/activation/elu_activation.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Factivation%2Felu_activation.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "ELU: a floor, not a cliff",
+ "goal": "How ELU keeps a soft negative side with a floor at -alpha, and why its slope never quite dies.",
+ "text": "ELU keeps the positive side of ReLU and replaces the cliff on the negative side with alpha (e^x - 1), a curve that flattens toward -alpha. With alpha 1, x = -3 gives -0.9502, already near the floor of -1, and the slope there is 0.0498: small, but not 0, so the unit can still move. Because the negative outputs are not thrown away, a layer's outputs average closer to 0. The spec elu_activation.t27 computes the curve, the slope and the floor, and its own batch at -1, 0 and 2: -0.632, 0 and 2.",
+ "task": "Set alpha to 0.5 and read the floor; then find how far left x must go before the slope is under 0.01."
+ },
+ "ru": {
+ "title": "ELU: пол, а не обрыв",
+ "goal": "Как ELU сохраняет мягкую отрицательную сторону с полом на -alpha и почему её наклон никогда не умирает до конца.",
+ "text": "ELU сохраняет положительную сторону ReLU и заменяет обрыв на отрицательной стороне кривой alpha (e^x - 1), которая выполаживается к -alpha. При alpha 1 вход x = -3 даёт -0.9502, уже близко к полу -1, а наклон там 0.0498: мал, но не 0, так что нейрон ещё может сдвинуться. Отрицательные выходы не выбрасываются, поэтому среднее выходов слоя ближе к 0. Спека elu_activation.t27 вычисляет кривую, наклон и пол, а также свой пакет при -1, 0 и 2: -0.632, 0 и 2.",
+ "task": "Поставьте alpha на 0.5 и прочитайте пол; затем найдите, как далеко влево должен уйти x, чтобы наклон стал меньше 0.01."
+ }
+ },
+ {
+ "id": "softmax-and-temperature",
+ "n": 7,
+ "module": "a-layer",
+ "widget": {
+ "id": "nn-softmax",
+ "kind": "tool",
+ "title": "Softmax: how a network turns scores into a choice",
+ "url": "https://t27.ai/widgets/nn-softmax/",
+ "preview": "widgets/nn-softmax/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-embedding",
+ "kind": "tool",
+ "title": "Embeddings: how a word becomes a row of numbers",
+ "url": "https://t27.ai/widgets/nn-embedding/",
+ "preview": "widgets/nn-embedding/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-dropout",
+ "kind": "tool",
+ "title": "Dropout: training a network with neurons switched off",
+ "url": "https://t27.ai/widgets/nn-dropout/",
+ "preview": "widgets/nn-dropout/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-bce",
+ "kind": "tool",
+ "title": "Cross-entropy: why a confident wrong answer costs so much",
+ "url": "https://t27.ai/widgets/nn-bce/",
+ "preview": "widgets/nn-bce/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/activation/softmax.t27",
+ "source": "t27/files/specs/ml/activation/softmax.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Factivation%2Fsoftmax.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Softmax and temperature",
+ "goal": "How softmax turns scores into probabilities that sum to one, and what the temperature does to them.",
+ "text": "A classifier ends with one score per class, and softmax turns the scores into probabilities: exp of each score over the sum of all of them. Scores 1, 2 and 3 become 0.090, 0.245 and 0.665. Dividing the scores by a temperature first changes how sure it is: at 0.5 the top score takes 0.867, at 5 the three are 0.27, 0.33 and 0.40. A language model picks its next token from exactly this, and the temperature setting of a chat model is this T. The spec softmax.t27 subtracts the largest score first, so exp never overflows.",
+ "task": "Cool the temperature to 0.1 and read the three probabilities; then make a tie and heat it."
+ },
+ "ru": {
+ "title": "Softmax и температура",
+ "goal": "Как softmax превращает оценки в вероятности с суммой один и что с ними делает температура.",
+ "text": "Классификатор заканчивается одной оценкой на класс, и softmax превращает оценки в вероятности: exp каждой оценки, делённая на сумму всех. Оценки 1, 2 и 3 становятся 0.090, 0.245 и 0.665. Если сначала разделить оценки на температуру, меняется уверенность: при 0.5 верхняя оценка забирает 0.867, при 5 все три равны 0.27, 0.33 и 0.40. Языковая модель выбирает следующий токен именно отсюда, и настройка температуры чат-модели это и есть T. Спека softmax.t27 сначала вычитает наибольшую оценку, поэтому exp никогда не переполняется.",
+ "task": "Охладите температуру до 0.1 и прочитайте три вероятности; затем сделайте ничью и нагрейте её."
+ }
+ },
+ {
+ "id": "embeddings",
+ "n": 8,
+ "module": "a-layer",
+ "widget": {
+ "id": "nn-embedding",
+ "kind": "tool",
+ "title": "Embeddings: how a word becomes a row of numbers",
+ "url": "https://t27.ai/widgets/nn-embedding/",
+ "preview": "widgets/nn-embedding/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-softmax",
+ "kind": "tool",
+ "title": "Softmax: how a network turns scores into a choice",
+ "url": "https://t27.ai/widgets/nn-softmax/",
+ "preview": "widgets/nn-softmax/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-dropout",
+ "kind": "tool",
+ "title": "Dropout: training a network with neurons switched off",
+ "url": "https://t27.ai/widgets/nn-dropout/",
+ "preview": "widgets/nn-dropout/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-positions",
+ "kind": "tool",
+ "title": "Positional encoding: how a transformer knows word order",
+ "url": "https://t27.ai/widgets/nn-positions/",
+ "preview": "widgets/nn-positions/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/layers/embedding_layer.t27",
+ "source": "t27/files/specs/ml/layers/embedding_layer.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Flayers%2Fembedding_layer.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Embeddings",
+ "goal": "How a token becomes a row of numbers, and why learning changes only the rows that were read.",
+ "text": "A network cannot multiply a word, so each token is an index into a table with one row of numbers per token, and looking a token up copies its row. In the spec the table has 4 tokens of width 2 and row v holds (10 v, 10 v + 1), so token 2 reads (20, 21). Learning sends the gradient back only to the rows that were read, and a token read twice collects both: gradient rows (1, 2) and (3, 4) on token 2 add up to (4, 6). The spec embedding_layer.t27 computes the lookup and the gradient.",
+ "task": "Read token 2 twice and then tokens 1 and 3; count the rows whose gradient is not 0 each time."
+ },
+ "ru": {
+ "title": "Эмбеддинги",
+ "goal": "Как токен превращается в строку чисел и почему обучение меняет только прочитанные строки.",
+ "text": "Сеть не умеет умножать слово, поэтому каждый токен это индекс в таблице с одной строкой чисел на токен, а поиск токена копирует его строку. В спеке в таблице 4 токена шириной 2, и строка v хранит (10 v, 10 v + 1), так что токен 2 читает (20, 21). Обучение отправляет градиент назад только в прочитанные строки, а токен, прочитанный дважды, собирает оба: строки градиента (1, 2) и (3, 4) на токене 2 складываются в (4, 6). Спека embedding_layer.t27 вычисляет поиск и градиент.",
+ "task": "Прочитайте токен 2 дважды, а затем токены 1 и 3; каждый раз сосчитайте строки, где градиент не 0."
+ }
+ },
+ {
+ "id": "dropout",
+ "n": 9,
+ "module": "a-layer",
+ "widget": {
+ "id": "nn-dropout",
+ "kind": "tool",
+ "title": "Dropout: training a network with neurons switched off",
+ "url": "https://t27.ai/widgets/nn-dropout/",
+ "preview": "widgets/nn-dropout/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-softmax",
+ "kind": "tool",
+ "title": "Softmax: how a network turns scores into a choice",
+ "url": "https://t27.ai/widgets/nn-softmax/",
+ "preview": "widgets/nn-softmax/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-embedding",
+ "kind": "tool",
+ "title": "Embeddings: how a word becomes a row of numbers",
+ "url": "https://t27.ai/widgets/nn-embedding/",
+ "preview": "widgets/nn-embedding/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/layers/dropout_layer.t27",
+ "source": "t27/files/specs/ml/layers/dropout_layer.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Flayers%2Fdropout_layer.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Dropout",
+ "goal": "Why switching units off while training helps, and why the survivors must be scaled.",
+ "text": "While training, dropout switches off a share p of the units at random, so the rest cannot lean on any one of them. The units that survive must then be scaled, or the next layer would see the signal at another size in training than at inference. Inverted dropout scales them by 1 / (1 - p) while training: with p = 0.5, input 1, 2, 3, 4 becomes 2, 0, 6, 0, and inference is a plain pass. Classic dropout keeps 1, 0, 3, 0 and scales by 0.5 at inference instead. The spec dropout_layer.t27 uses a fixed mask, keep, drop, keep, drop, so the numbers are the same every time.",
+ "task": "Compare the outputs of the four modes; then read the gradient of a dropped unit in training."
+ },
+ "ru": {
+ "title": "Dropout: выключенные нейроны",
+ "goal": "Почему выключение нейронов во время обучения помогает и почему выживших нужно масштабировать.",
+ "text": "Во время обучения dropout случайно выключает долю p нейронов, чтобы остальные не опирались ни на один из них. Выжившие нейроны тогда нужно масштабировать, иначе следующий слой видел бы сигнал другого размера при обучении, чем при выводе. Обратный dropout умножает их на 1 / (1 - p) при обучении: с p = 0.5 вход 1, 2, 3, 4 становится 2, 0, 6, 0, а вывод это простой проход. Классический dropout оставляет 1, 0, 3, 0 и умножает на 0.5 уже при выводе. Спека dropout_layer.t27 использует фиксированную маску, оставить, выбросить, оставить, выбросить, поэтому числа каждый раз одинаковые.",
+ "task": "Сравните выходы четырёх режимов; затем прочитайте градиент выброшенного нейрона при обучении."
+ }
+ },
+ {
+ "id": "patches-of-a-picture",
+ "n": 10,
+ "module": "pictures",
+ "widget": {
+ "id": "nn-conv",
+ "kind": "tool",
+ "title": "Patches of a picture: a 2 x 2 window, averaged",
+ "url": "https://t27.ai/widgets/nn-conv/",
+ "preview": "widgets/nn-conv/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-maxpool",
+ "kind": "tool",
+ "title": "Max pooling: what a network keeps when it shrinks a picture",
+ "url": "https://t27.ai/widgets/nn-maxpool/",
+ "preview": "widgets/nn-maxpool/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-batchnorm",
+ "kind": "tool",
+ "title": "Batch norm: features on very different scales, made comparable",
+ "url": "https://t27.ai/widgets/nn-batchnorm/",
+ "preview": "widgets/nn-batchnorm/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "tc-detector",
+ "kind": "tool",
+ "title": "A bar detector: one neuron, four patterns",
+ "url": "https://t27.ai/widgets/tc-detector/",
+ "preview": "widgets/tc-detector/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/layers/avgpool2d_layer.t27",
+ "source": "t27/files/specs/ml/layers/avgpool2d_layer.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Flayers%2Favgpool2d_layer.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Patches of a picture",
+ "goal": "Why networks read a picture in patches, and what a 2 x 2 window averaged over each patch gives.",
+ "text": "A picture is a grid of numbers, and networks read it in small patches so that the same detector works anywhere in it. Here a 2 x 2 window steps by 2 over a 4 x 4 image whose cell i holds i, and each output is the average of its patch: 2.5, 4.5, 10.5 and 12.5. In the backward pass each output's gradient is split equally over its 4 cells. This is average pooling: a convolution would weight the 4 cells with a learned kernel, and its spec, t27#10612, is not merged yet. The spec avgpool2d_layer.t27 computes both passes.",
+ "task": "Tap each patch and check its average by eye; then switch to the gradient and read what each cell of window 4 receives."
+ },
+ "ru": {
+ "title": "Участки картинки",
+ "goal": "Почему сети читают картинку по участкам и что даёт окно 2 x 2, усреднённое по каждому участку.",
+ "text": "Картинка это сетка чисел, и сети читают её небольшими участками, чтобы один и тот же детектор работал в любом её месте. Здесь окно 2 x 2 шагает по 2 по картинке 4 x 4, где в клетке i лежит i, и каждый выход это среднее своего участка: 2.5, 4.5, 10.5 и 12.5. При обратном проходе градиент каждого выхода делится поровну на его 4 клетки. Это усредняющий пулинг: свёртка взвешивала бы 4 клетки выученным ядром, и её спека, t27#10612, ещё не слита. Спека avgpool2d_layer.t27 вычисляет оба прохода.",
+ "task": "Нажмите на каждый участок и проверьте его среднее на глаз; затем включите градиент и прочитайте, что получает каждая клетка окна 4."
+ }
+ },
+ {
+ "id": "max-pooling",
+ "n": 11,
+ "module": "pictures",
+ "widget": {
+ "id": "nn-maxpool",
+ "kind": "tool",
+ "title": "Max pooling: what a network keeps when it shrinks a picture",
+ "url": "https://t27.ai/widgets/nn-maxpool/",
+ "preview": "widgets/nn-maxpool/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-conv",
+ "kind": "tool",
+ "title": "Patches of a picture: a 2 x 2 window, averaged",
+ "url": "https://t27.ai/widgets/nn-conv/",
+ "preview": "widgets/nn-conv/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-batchnorm",
+ "kind": "tool",
+ "title": "Batch norm: features on very different scales, made comparable",
+ "url": "https://t27.ai/widgets/nn-batchnorm/",
+ "preview": "widgets/nn-batchnorm/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/layers/maxpool2d_layer.t27",
+ "source": "t27/files/specs/ml/layers/maxpool2d_layer.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Flayers%2Fmaxpool2d_layer.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Max pooling",
+ "goal": "What max pooling keeps, where it remembers taking it from, and where the gradient goes.",
+ "text": "Max pooling keeps the largest value of each window and drops the rest, so the network cares less about exactly where a feature sits. On the spec's 4 x 4 grid, 2 x 2 windows with stride 2 keep 4, 8, 9 and -2, taken from cells 5, 6, 13 and 11. Overlapping 3 x 3 windows with stride 1 keep 8, 8, 9 and 9. In the backward pass each window's whole gradient goes to its maximum: gradients 1, 2, 3 and 4 land on cells 5, 6, 13 and 11, and every other cell gets 0. The spec maxpool2d_layer.t27 computes the outputs, their positions and the gradient.",
+ "task": "Switch to 3 x 3 windows and find the cell two outputs share; then turn on the gradient for 2 x 2 windows."
+ },
+ "ru": {
+ "title": "Max pooling: максимум окна",
+ "goal": "Что оставляет max pooling, откуда, как он помнит, это взято и куда идёт градиент.",
+ "text": "Max pooling оставляет наибольшее значение каждого окна и отбрасывает остальные, поэтому сети становится менее важно, где именно лежит признак. На сетке спеки 4 x 4 окна 2 x 2 с шагом 2 оставляют 4, 8, 9 и -2, взятые из клеток 5, 6, 13 и 11. Перекрывающиеся окна 3 x 3 с шагом 1 оставляют 8, 8, 9 и 9. При обратном проходе весь градиент окна идёт в его максимум: градиенты 1, 2, 3 и 4 попадают в клетки 5, 6, 13 и 11, а все остальные клетки получают 0. Спека maxpool2d_layer.t27 вычисляет выходы, их позиции и градиент.",
+ "task": "Переключитесь на окна 3 x 3 и найдите клетку, общую для двух выходов; затем включите градиент для окон 2 x 2."
+ }
+ },
+ {
+ "id": "batch-normalisation",
+ "n": 12,
+ "module": "pictures",
+ "widget": {
+ "id": "nn-batchnorm",
+ "kind": "tool",
+ "title": "Batch norm: features on very different scales, made comparable",
+ "url": "https://t27.ai/widgets/nn-batchnorm/",
+ "preview": "widgets/nn-batchnorm/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-conv",
+ "kind": "tool",
+ "title": "Patches of a picture: a 2 x 2 window, averaged",
+ "url": "https://t27.ai/widgets/nn-conv/",
+ "preview": "widgets/nn-conv/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-maxpool",
+ "kind": "tool",
+ "title": "Max pooling: what a network keeps when it shrinks a picture",
+ "url": "https://t27.ai/widgets/nn-maxpool/",
+ "preview": "widgets/nn-maxpool/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-encoder",
+ "kind": "tool",
+ "title": "An encoder block: what one transformer layer does",
+ "url": "https://t27.ai/widgets/nn-encoder/",
+ "preview": "widgets/nn-encoder/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/layers/batchnorm_layer.t27",
+ "source": "t27/files/specs/ml/layers/batchnorm_layer.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Flayers%2Fbatchnorm_layer.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Batch normalisation",
+ "goal": "How batch norm puts features on different scales on one scale, and what it uses at inference.",
+ "text": "Features on different scales pull a network around: in the spec's batch one feature is ten times another. Batch normalisation subtracts each feature's batch mean and divides by its spread, so the features 1, 2, 3, 4 and 10, 20, 30, 40 both come out as -1.34, -0.45, 0.45 and 1.34, and -1, -1, 1, 1 comes out as -1 and 1. A learned gamma and beta then scale and shift it back where that helps. At inference the layer uses running averages instead of the batch: with mean 0, variance 1, gamma 1 and beta 0.25, the inputs 1, 2 and 3 give 1.25, 2.25 and 3.25. The spec batchnorm_layer.t27 computes both.",
+ "task": "Compare the two normalised columns; then set the running mean 2 and variance 4 and read the inference outputs."
+ },
+ "ru": {
+ "title": "Пакетная нормализация",
+ "goal": "Как пакетная нормализация приводит признаки разного масштаба к одному и чем она пользуется при выводе.",
+ "text": "Признаки разного масштаба дёргают сеть: в пакете спеки один признак в десять раз больше другого. Пакетная нормализация вычитает среднее каждого признака по пакету и делит на его разброс, поэтому признаки 1, 2, 3, 4 и 10, 20, 30, 40 оба выходят как -1.34, -0.45, 0.45 и 1.34, а -1, -1, 1, 1 выходит как -1 и 1. Затем выученные gamma и beta масштабируют и сдвигают его обратно, где это полезно. При выводе слой вместо пакета использует скользящие средние: при среднем 0, дисперсии 1, gamma 1 и beta 0.25 входы 1, 2 и 3 дают 1.25, 2.25 и 3.25. Спека batchnorm_layer.t27 вычисляет оба режима.",
+ "task": "Сравните два нормализованных столбца; затем поставьте скользящее среднее 2 и дисперсию 4 и прочитайте выходы вывода."
+ }
+ },
+ {
+ "id": "mean-squared-error",
+ "n": 13,
+ "module": "the-error",
+ "widget": {
+ "id": "nn-mse",
+ "kind": "tool",
+ "title": "Mean squared error: how wrong a guess is, as one number",
+ "url": "https://t27.ai/widgets/nn-mse/",
+ "preview": "widgets/nn-mse/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-bce",
+ "kind": "tool",
+ "title": "Cross-entropy: why a confident wrong answer costs so much",
+ "url": "https://t27.ai/widgets/nn-bce/",
+ "preview": "widgets/nn-bce/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-huber",
+ "kind": "tool",
+ "title": "Huber loss: a loss that does not panic at an outlier",
+ "url": "https://t27.ai/widgets/nn-huber/",
+ "preview": "widgets/nn-huber/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/loss/mse_loss.t27",
+ "source": "t27/files/specs/ml/loss/mse_loss.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Floss%2Fmse_loss.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Mean squared error",
+ "goal": "How MSE turns every miss into one number, why it chases large errors, and where its gradient points.",
+ "text": "A loss turns all the misses of a model into one number to push down. Mean squared error averages the squared misses: predictions 1.5, 2 and 2 against targets 1, 2 and 3 cost (0.25 + 0 + 1) / 3 = 0.4167. Squaring makes a miss of 2 cost 4 times a miss of 1, so MSE chases large errors. Its gradient for each prediction is 2 (p - t) / n, here 0.333, 0 and -0.667: each prediction is pushed back toward its target. As a function of one prediction the loss is a parabola, a bowl with one bottom. The spec mse_loss.t27 computes the loss and the gradient.",
+ "task": "Move prediction 2 onto its target and read the loss; then drag along the bowl and find its bottom."
+ },
+ "ru": {
+ "title": "Среднеквадратичная ошибка",
+ "goal": "Как MSE превращает каждый промах в одно число, почему она гоняется за большими ошибками и куда смотрит её градиент.",
+ "text": "Функция потерь превращает все промахи модели в одно число, которое нужно уменьшать. Среднеквадратичная ошибка усредняет квадраты промахов: предсказания 1.5, 2 и 2 против целей 1, 2 и 3 стоят (0.25 + 0 + 1) / 3 = 0.4167. Из-за квадрата промах на 2 стоит в 4 раза больше промаха на 1, поэтому MSE гоняется за большими ошибками. Её градиент для каждого предсказания равен 2 (p - t) / n, здесь 0.333, 0 и -0.667: каждое предсказание толкается обратно к своей цели. Как функция одного предсказания потеря это парабола, чаша с одним дном. Спека mse_loss.t27 вычисляет потерю и градиент.",
+ "task": "Передвиньте предсказание 2 на его цель и прочитайте потерю; затем ведите по чаше и найдите её дно."
+ }
+ },
+ {
+ "id": "cross-entropy",
+ "n": 14,
+ "module": "the-error",
+ "widget": {
+ "id": "nn-bce",
+ "kind": "tool",
+ "title": "Cross-entropy: why a confident wrong answer costs so much",
+ "url": "https://t27.ai/widgets/nn-bce/",
+ "preview": "widgets/nn-bce/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-mse",
+ "kind": "tool",
+ "title": "Mean squared error: how wrong a guess is, as one number",
+ "url": "https://t27.ai/widgets/nn-mse/",
+ "preview": "widgets/nn-mse/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-huber",
+ "kind": "tool",
+ "title": "Huber loss: a loss that does not panic at an outlier",
+ "url": "https://t27.ai/widgets/nn-huber/",
+ "preview": "widgets/nn-huber/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-sigmoid",
+ "kind": "tool",
+ "title": "Sigmoid: turn any number into a probability",
+ "url": "https://t27.ai/widgets/nn-sigmoid/",
+ "preview": "widgets/nn-sigmoid/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/loss/binary_crossentropy_loss.t27",
+ "source": "t27/files/specs/ml/loss/binary_crossentropy_loss.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Floss%2Fbinary_crossentropy_loss.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Cross-entropy",
+ "goal": "Why cross-entropy charges a confident wrong answer so much, and why that is what trains a classifier.",
+ "text": "A classifier outputs a probability, and cross-entropy charges -log of the probability it gave to the true answer. Predictions 0.8, 0.3 and 0.9 against answers 1, 0 and 1 cost 0.2284 on average. Being sure and right is nearly free: 0.99, 0.01 and 0.99 cost 0.0101. Being sure and wrong is expensive: 0.01, 0.99 and 0.01 cost 4.6052, and toward 0 the cost grows without bound, which is why the spec clips p to [1e-7, 1 - 1e-7]. Its gradient is largest exactly where the model is sure and wrong. The spec binary_crossentropy_loss.t27 computes the loss and the gradient, from probabilities or from logits.",
+ "task": "Drag prediction 0 toward 0.01 and watch the curve; then read the gradient bars at 0.5, 0.5 and 0.5."
+ },
+ "ru": {
+ "title": "Перекрёстная энтропия",
+ "goal": "Почему перекрёстная энтропия так дорого берёт за уверенный неверный ответ и почему именно это учит классификатор.",
+ "text": "Классификатор выдаёт вероятность, а перекрёстная энтропия берёт -log вероятности, которую он дал верному ответу. Предсказания 0.8, 0.3 и 0.9 против ответов 1, 0 и 1 стоят в среднем 0.2284. Быть уверенным и правым почти бесплатно: 0.99, 0.01 и 0.99 стоят 0.0101. Быть уверенным и неправым дорого: 0.01, 0.99 и 0.01 стоят 4.6052, а к 0 стоимость растёт без предела, поэтому спека обрезает p до [1e-7, 1 - 1e-7]. Её градиент наибольший как раз там, где модель уверена и неправа. Спека binary_crossentropy_loss.t27 вычисляет потерю и градиент, из вероятностей или из логитов.",
+ "task": "Тяните предсказание 0 к 0.01 и смотрите на кривую; затем прочитайте столбики градиента при 0.5, 0.5 и 0.5."
+ }
+ },
+ {
+ "id": "the-huber-loss",
+ "n": 15,
+ "module": "the-error",
+ "widget": {
+ "id": "nn-huber",
+ "kind": "tool",
+ "title": "Huber loss: a loss that does not panic at an outlier",
+ "url": "https://t27.ai/widgets/nn-huber/",
+ "preview": "widgets/nn-huber/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-mse",
+ "kind": "tool",
+ "title": "Mean squared error: how wrong a guess is, as one number",
+ "url": "https://t27.ai/widgets/nn-mse/",
+ "preview": "widgets/nn-mse/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-bce",
+ "kind": "tool",
+ "title": "Cross-entropy: why a confident wrong answer costs so much",
+ "url": "https://t27.ai/widgets/nn-bce/",
+ "preview": "widgets/nn-bce/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/loss/huber_loss.t27",
+ "source": "t27/files/specs/ml/loss/huber_loss.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Floss%2Fhuber_loss.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "The Huber loss",
+ "goal": "How the Huber loss treats a small error like squared error and a large one linearly, so outliers pull less.",
+ "text": "Squared error lets one outlier dominate: an error of 3 costs 4.5 as half its square, as much as 36 errors of 0.5. Huber is half the square while |e| is within delta and then grows only linearly, delta |e| - delta^2 / 2, so with delta 1 that error costs 2.5. The two pieces meet with the same value and slope, and the gradient never grows past delta: a far-off point pulls on the model with a bounded force. For the spec's pair of errors 0.5 and 3 the mean is 1.3125, against 2.3125 for half the square. The spec huber_loss.t27 computes both.",
+ "task": "Set delta to 0.5 and drag the error to 3; then find where the two curves part."
+ },
+ "ru": {
+ "title": "Функция Хьюбера",
+ "goal": "Как функция Хьюбера обходится с малой ошибкой как с квадратичной, а с большой линейно, чтобы выбросы тянули слабее.",
+ "text": "Квадратичная ошибка позволяет одному выбросу всё решать: ошибка 3 стоит 4.5 как половина своего квадрата, столько же, сколько 36 ошибок по 0.5. Функция Хьюбера равна половине квадрата, пока |e| не больше delta, а дальше растёт только линейно, delta |e| - delta^2 / 2, так что при delta 1 та же ошибка стоит 2.5. Два куска сходятся с одинаковым значением и наклоном, и градиент никогда не превышает delta: далёкая точка тянет модель с ограниченной силой. Для пары ошибок спеки 0.5 и 3 среднее равно 1.3125 против 2.3125 для половины квадрата. Спека huber_loss.t27 вычисляет обе.",
+ "task": "Поставьте delta на 0.5 и тяните ошибку к 3; затем найдите, где две кривые расходятся."
+ }
+ },
+ {
+ "id": "gradient-descent",
+ "n": 16,
+ "module": "downhill",
+ "widget": {
+ "id": "nn-sgd",
+ "kind": "tool",
+ "title": "Gradient descent: how a network walks downhill",
+ "url": "https://t27.ai/widgets/nn-sgd/",
+ "preview": "widgets/nn-sgd/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-rmsprop",
+ "kind": "tool",
+ "title": "RMSProp: a step that does not care how big the gradient is",
+ "url": "https://t27.ai/widgets/nn-rmsprop/",
+ "preview": "widgets/nn-rmsprop/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-adam",
+ "kind": "tool",
+ "title": "Adam: the optimizer most networks are trained with",
+ "url": "https://t27.ai/widgets/nn-adam/",
+ "preview": "widgets/nn-adam/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-mse",
+ "kind": "tool",
+ "title": "Mean squared error: how wrong a guess is, as one number",
+ "url": "https://t27.ai/widgets/nn-mse/",
+ "preview": "widgets/nn-mse/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/optimizer/sgd.t27",
+ "source": "t27/files/specs/ml/optimizer/sgd.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Foptimizer%2Fsgd.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Gradient descent",
+ "goal": "How gradient descent walks down a loss, what momentum adds, and why too large a rate never settles.",
+ "text": "Training is walking downhill on the loss. Gradient descent moves a weight by -lr times its gradient each step; in the spec the loss is (x - 3)^2 from x = 0, and lr 0.1 reaches 2.99996 in 50 steps. Momentum adds a share of the last move, which speeds up a long slope and overshoots a sharp bottom: at 0.9 the weight shoots past 3 to about 5.13 and rings down to 3.09 after 50 steps. Nesterov takes the gradient one step ahead and swings less, to about 4.25. Too large a rate never settles: at lr 1 the weight bounces between 0 and 6 for good. The spec sgd.t27 computes every step.",
+ "task": "Find the largest learning rate that still settles without momentum; then add momentum 0.9 and count the swings."
+ },
+ "ru": {
+ "title": "Градиентный спуск",
+ "goal": "Как градиентный спуск идёт вниз по функции потерь, что добавляет момент и почему слишком большой шаг никогда не успокаивается.",
+ "text": "Обучение это спуск под гору по функции потерь. Градиентный спуск каждый шаг сдвигает вес на -lr, умноженное на его градиент; в спеке потеря равна (x - 3)^2 от x = 0, и lr 0.1 доходит до 2.99996 за 50 шагов. Момент добавляет долю прошлого сдвига, что ускоряет длинный склон и перелетает острое дно: при 0.9 вес пролетает 3 примерно до 5.13 и, раскачиваясь, приходит к 3.09 после 50 шагов. Нестеров берёт градиент на шаг вперёд и качается меньше, примерно до 4.25. Слишком большой шаг никогда не успокаивается: при lr 1 вес навсегда скачет между 0 и 6. Спека sgd.t27 вычисляет каждый шаг.",
+ "task": "Найдите наибольший шаг обучения, который ещё успокаивается без момента; затем добавьте момент 0.9 и сосчитайте качания."
+ }
+ },
+ {
+ "id": "rmsprop",
+ "n": 17,
+ "module": "downhill",
+ "widget": {
+ "id": "nn-rmsprop",
+ "kind": "tool",
+ "title": "RMSProp: a step that does not care how big the gradient is",
+ "url": "https://t27.ai/widgets/nn-rmsprop/",
+ "preview": "widgets/nn-rmsprop/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-sgd",
+ "kind": "tool",
+ "title": "Gradient descent: how a network walks downhill",
+ "url": "https://t27.ai/widgets/nn-sgd/",
+ "preview": "widgets/nn-sgd/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-adam",
+ "kind": "tool",
+ "title": "Adam: the optimizer most networks are trained with",
+ "url": "https://t27.ai/widgets/nn-adam/",
+ "preview": "widgets/nn-adam/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/optimizer/rmsprop.t27",
+ "source": "t27/files/specs/ml/optimizer/rmsprop.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Foptimizer%2Frmsprop.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "RMSProp",
+ "goal": "How RMSProp divides each step by a running root mean square, so the step no longer depends on the gradient's size.",
+ "text": "The right step size depends on the size of the gradient, and different weights see gradients a million times apart. RMSProp keeps a running mean of the squared gradient, the cache, and divides each step by its root, so the step has no units: with a constant gradient of 0.001, 0.5 or 1000 the weight moves by exactly the same 0.0316 on step 1 and walks the same path, reaching -0.0914 after 100 steps at lr 0.01. The decay sets how slowly the cache fills: at 0.99 the same 100 steps end at -1.0261. The spec rmsprop.t27 computes the weight and the cache.",
+ "task": "Switch between the three gradients and check that the lines do not move; then raise the decay to 0.99."
+ },
+ "ru": {
+ "title": "RMSProp: шаг без масштаба",
+ "goal": "Как RMSProp делит каждый шаг на скользящий корень среднего квадрата, чтобы шаг больше не зависел от величины градиента.",
+ "text": "Правильный размер шага зависит от величины градиента, а разные веса видят градиенты, отличающиеся в миллион раз. RMSProp хранит скользящее среднее квадрата градиента, кэш, и делит каждый шаг на его корень, поэтому у шага нет единиц: при постоянном градиенте 0.001, 0.5 или 1000 вес на шаге 1 сдвигается ровно на те же 0.0316 и идёт тем же путём, доходя до -0.0914 после 100 шагов при lr 0.01. Затухание задаёт, как медленно наполняется кэш: при 0.99 те же 100 шагов заканчиваются на -1.0261. Спека rmsprop.t27 вычисляет вес и кэш.",
+ "task": "Переключайтесь между тремя градиентами и убедитесь, что линии не двигаются; затем поднимите затухание до 0.99."
+ }
+ },
+ {
+ "id": "adam",
+ "n": 18,
+ "module": "downhill",
+ "widget": {
+ "id": "nn-adam",
+ "kind": "tool",
+ "title": "Adam: the optimizer most networks are trained with",
+ "url": "https://t27.ai/widgets/nn-adam/",
+ "preview": "widgets/nn-adam/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-sgd",
+ "kind": "tool",
+ "title": "Gradient descent: how a network walks downhill",
+ "url": "https://t27.ai/widgets/nn-sgd/",
+ "preview": "widgets/nn-sgd/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-rmsprop",
+ "kind": "tool",
+ "title": "RMSProp: a step that does not care how big the gradient is",
+ "url": "https://t27.ai/widgets/nn-rmsprop/",
+ "preview": "widgets/nn-rmsprop/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-lamb",
+ "kind": "tool",
+ "title": "LAMB: one learning rate for layers of every size",
+ "url": "https://t27.ai/widgets/nn-lamb/",
+ "preview": "widgets/nn-lamb/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/optimizer/adam.t27",
+ "source": "t27/files/specs/ml/optimizer/adam.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Foptimizer%2Fadam.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Adam",
+ "goal": "How Adam combines momentum with a running scale, and why its bias correction makes step 1 a fixed size.",
+ "text": "Adam combines the two ideas: m, a running mean of the gradient, as in momentum, and v, a running mean of its square, as in RMSProp; the step is lr m / sqrt(v). Both start at 0 and are too small early on, so Adam divides them by 1 - beta^t, and on step 1 that makes the move exactly lr times the sign of the gradient: gradients 2, -40 and 0.5 all move their weights by 0.001. On the spec's bowl (x - 3)^2 at lr 0.1, Adam rises from 0, overshoots to about 3.18 and settles at 3.00005 by step 200. The spec adam.t27 computes every step.",
+ "task": "Give the three gradients very different sizes and read the weights after step 1; then find where the path tops out."
+ },
+ "ru": {
+ "title": "Adam: момент и масштаб",
+ "goal": "Как Adam соединяет момент со скользящим масштабом и почему поправка смещения делает шаг 1 фиксированного размера.",
+ "text": "Adam соединяет обе идеи: m, скользящее среднее градиента, как в моменте, и v, скользящее среднее его квадрата, как в RMSProp; шаг равен lr m / sqrt(v). Оба начинают с 0 и поначалу слишком малы, поэтому Adam делит их на 1 - beta^t, и на шаге 1 это делает сдвиг ровно lr, умноженным на знак градиента: градиенты 2, -40 и 0.5 сдвигают свои веса на одинаковые 0.001. На чаше спеки (x - 3)^2 при lr 0.1 Adam поднимается от 0, перелетает примерно до 3.18 и успокаивается на 3.00005 к шагу 200. Спека adam.t27 вычисляет каждый шаг.",
+ "task": "Дайте трём градиентам очень разные размеры и прочитайте веса после шага 1; затем найдите, где путь достигает вершины."
+ }
+ },
+ {
+ "id": "backpropagation",
+ "n": 19,
+ "module": "learning",
+ "widget": {
+ "id": "nn-backprop",
+ "kind": "tool",
+ "title": "Backpropagation: how blame flows back through a layer",
+ "url": "https://t27.ai/widgets/nn-backprop/",
+ "preview": "widgets/nn-backprop/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-xor",
+ "kind": "tool",
+ "title": "XOR: a network learns what one neuron cannot",
+ "url": "https://t27.ai/widgets/nn-xor/",
+ "preview": "widgets/nn-xor/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-lamb",
+ "kind": "tool",
+ "title": "LAMB: one learning rate for layers of every size",
+ "url": "https://t27.ai/widgets/nn-lamb/",
+ "preview": "widgets/nn-lamb/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "tc-tiny-layer",
+ "kind": "tool",
+ "title": "A layer of three ternary neurons",
+ "url": "https://t27.ai/widgets/tc-tiny-layer/",
+ "preview": "widgets/tc-tiny-layer/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/layers/dense_layer.t27",
+ "source": "t27/files/specs/ml/layers/dense_layer.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Flayers%2Fdense_layer.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Backpropagation",
+ "goal": "How the gradient of each weight and each input of a layer follows from the gradient of its outputs.",
+ "text": "To learn, every weight needs to know how the loss changes when it moves. Backpropagation answers one layer at a time: given g, how much the loss wants each output to move, the weight from input i to output o gets g_o x_i, and input i gets the sum over o of W[o][i] g_o, which it hands on to the layer below. With inputs (1, 2, 3) and g = (1, 0.5) the weight gradients are 1, 2, 3 and 0.5, 1, 1.5, and the input gradients 0.5, 2 and 3.5. The layer is linear, so moving x0 by 1 moves the loss g . y from 15.25 to 15.75, by its gradient 0.5. The spec dense_layer.t27 computes all three.",
+ "task": "Set g = (1, 0) and read which edges go dark; then set g = (-2, 1) and check the input gradients by hand."
+ },
+ "ru": {
+ "title": "Обратное распространение",
+ "goal": "Как градиент каждого веса и каждого входа слоя следует из градиента его выходов.",
+ "text": "Чтобы учиться, каждому весу нужно знать, как меняется потеря, когда он сдвигается. Обратное распространение отвечает по одному слою за раз: при данном g, насколько потеря хочет сдвинуть каждый выход, вес от входа i к выходу o получает g_o x_i, а вход i получает сумму по o от W[o][i] g_o и передаёт её слою ниже. При входах (1, 2, 3) и g = (1, 0.5) градиенты весов равны 1, 2, 3 и 0.5, 1, 1.5, а градиенты входов 0.5, 2 и 3.5. Слой линейный, поэтому сдвиг x0 на 1 сдвигает потерю g . y с 15.25 до 15.75, на его градиент 0.5. Спека dense_layer.t27 вычисляет все три.",
+ "task": "Поставьте g = (1, 0) и посмотрите, какие рёбра гаснут; затем поставьте g = (-2, 1) и проверьте градиенты входов вручную."
+ }
+ },
+ {
+ "id": "learning-xor",
+ "n": 20,
+ "module": "learning",
+ "widget": {
+ "id": "nn-xor",
+ "kind": "tool",
+ "title": "XOR: a network learns what one neuron cannot",
+ "url": "https://t27.ai/widgets/nn-xor/",
+ "preview": "widgets/nn-xor/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-backprop",
+ "kind": "tool",
+ "title": "Backpropagation: how blame flows back through a layer",
+ "url": "https://t27.ai/widgets/nn-backprop/",
+ "preview": "widgets/nn-backprop/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-lamb",
+ "kind": "tool",
+ "title": "LAMB: one learning rate for layers of every size",
+ "url": "https://t27.ai/widgets/nn-lamb/",
+ "preview": "widgets/nn-lamb/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "tc-tiny-layer",
+ "kind": "tool",
+ "title": "A layer of three ternary neurons",
+ "url": "https://t27.ai/widgets/tc-tiny-layer/",
+ "preview": "widgets/tc-tiny-layer/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/pathway/mlp.t27",
+ "source": "t27/files/specs/ml/pathway/mlp.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Fpathway%2Fmlp.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Learning XOR",
+ "goal": "Why one neuron cannot learn XOR, and how a hidden layer trained by backpropagation does.",
+ "text": "XOR is 1 when exactly one of two inputs is 1. No straight line separates the cases that give 1 from those that give 0, so a single neuron cannot learn it. A hidden layer bends the space: the spec trains a 2-8-1 network of ReLUs by backpropagation, lr 0.05, from fixed initial weights. Its loss falls from 0.8896 in epoch 1 to 1.8e-13 after 2000 epochs, and its four answers come out within 4e-7 of 0, 1, 1 and 0. The widget runs that training in your browser, from scratch for every point of the curve. The spec mlp.t27 holds the network and the test that it learns.",
+ "task": "Drag the epochs back to 100 and read the four answers; then find the epoch where the curve stops falling."
+ },
+ "ru": {
+ "title": "Учим XOR",
+ "goal": "Почему один нейрон не может выучить XOR и как это делает скрытый слой, обученный обратным распространением.",
+ "text": "XOR равен 1, когда ровно один из двух входов равен 1. Никакая прямая не отделяет случаи, дающие 1, от дающих 0, поэтому один нейрон выучить его не может. Скрытый слой изгибает пространство: спека обучает сеть 2-8-1 из ReLU обратным распространением, lr 0.05, из фиксированных начальных весов. Её потеря падает с 0.8896 в эпохе 1 до 1.8e-13 после 2000 эпох, а четыре ответа выходят в пределах 4e-7 от 0, 1, 1 и 0. Виджет проводит это обучение в вашем браузере, заново для каждой точки кривой. Спека mlp.t27 хранит сеть и тест, что она учится.",
+ "task": "Отведите эпохи назад к 100 и прочитайте четыре ответа; затем найдите эпоху, где кривая перестаёт падать."
+ }
+ },
+ {
+ "id": "lamb-and-large-batches",
+ "n": 21,
+ "module": "learning",
+ "widget": {
+ "id": "nn-lamb",
+ "kind": "tool",
+ "title": "LAMB: one learning rate for layers of every size",
+ "url": "https://t27.ai/widgets/nn-lamb/",
+ "preview": "widgets/nn-lamb/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-adam",
+ "kind": "tool",
+ "title": "Adam: the optimizer most networks are trained with",
+ "url": "https://t27.ai/widgets/nn-adam/",
+ "preview": "widgets/nn-adam/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-xor",
+ "kind": "tool",
+ "title": "XOR: a network learns what one neuron cannot",
+ "url": "https://t27.ai/widgets/nn-xor/",
+ "preview": "widgets/nn-xor/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-backprop",
+ "kind": "tool",
+ "title": "Backpropagation: how blame flows back through a layer",
+ "url": "https://t27.ai/widgets/nn-backprop/",
+ "preview": "widgets/nn-backprop/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/optimizer/lamb.t27",
+ "source": "t27/files/specs/ml/optimizer/lamb.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Foptimizer%2Flamb.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "LAMB and large batches",
+ "goal": "How LAMB scales each layer's step by its own weight norm, so one learning rate suits layers of every size.",
+ "text": "Training with very large batches needs a large learning rate, and one rate can be too large for a small layer and too small for a big one. LAMB takes an Adam-style update for each layer and rescales it by the trust ratio, the layer's weight norm, capped at 10, over the update's norm, so every layer moves by the same fraction of itself. In the spec, the weights (1, 2) times a scale take a step whose norm is 0.001 times theirs: 0.00447 at scale 2, and once the weight norm passes 10, near scale 4.47, the step stays at 0.01. The spec lamb.t27 computes the update for one layer and for two at once.",
+ "task": "Read the step at scales 1 and 2 and compare; then drag past the bend and explain why the line goes flat."
+ },
+ "ru": {
+ "title": "LAMB и большие пакеты",
+ "goal": "Как LAMB масштабирует шаг каждого слоя его собственной нормой весов, чтобы один шаг обучения подходил слоям любого размера.",
+ "text": "Обучение на очень больших пакетах требует большого шага обучения, а один шаг может быть слишком большим для малого слоя и слишком малым для большого. LAMB берёт обновление в духе Adam для каждого слоя и перемасштабирует его коэффициентом доверия, нормой весов слоя, ограниченной 10, делённой на норму обновления, поэтому каждый слой сдвигается на одну и ту же долю самого себя. В спеке веса (1, 2), умноженные на масштаб, делают шаг с нормой, в 0.001 раз меньшей их собственной: 0.00447 при масштабе 2, а когда норма весов переходит 10, около масштаба 4.47, шаг остаётся на 0.01. Спека lamb.t27 вычисляет обновление для одного слоя и для двух сразу.",
+ "task": "Прочитайте шаг при масштабах 1 и 2 и сравните; затем перетащите за изгиб и объясните, почему линия становится плоской."
+ }
+ },
+ {
+ "id": "a-recurrent-cell",
+ "n": 22,
+ "module": "memory-and-attention",
+ "widget": {
+ "id": "nn-rnn",
+ "kind": "tool",
+ "title": "Recurrent cells: how a network remembers a moment ago",
+ "url": "https://t27.ai/widgets/nn-rnn/",
+ "preview": "widgets/nn-rnn/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-lstm",
+ "kind": "tool",
+ "title": "LSTM gates: how a cell decides what to keep",
+ "url": "https://t27.ai/widgets/nn-lstm/",
+ "preview": "widgets/nn-lstm/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-attention",
+ "kind": "tool",
+ "title": "Attention: how a word decides which other words to read",
+ "url": "https://t27.ai/widgets/nn-attention/",
+ "preview": "widgets/nn-attention/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/recurrent/rnn_cell.t27",
+ "source": "t27/files/specs/ml/recurrent/rnn_cell.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Frecurrent%2Frnn_cell.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "A recurrent cell",
+ "goal": "How a recurrent cell carries a state from step to step, and why that memory fades.",
+ "text": "A sentence comes one word at a time, and a recurrent cell carries a state h from one step to the next: h' = tanh(W_x x + W_h h + b). In the spec's tiny cell W_x is 1 and W_h 0.5, so a single pulse gives 0.7616, 0.3634 and 0.1797: the memory of it roughly halves each step. A second pulse at step 3 lifts h to 0.7968, a little above where the single pulse began, because the old memory is still there. After 8 steps only 0.0466 is left. That fading is why plain cells forget long contexts. The spec rnn_cell.t27 computes each step.",
+ "task": "Load one pulse and read how many steps it takes to fall under 0.1; then make the input alternate."
+ },
+ "ru": {
+ "title": "Рекуррентная ячейка",
+ "goal": "Как рекуррентная ячейка переносит состояние от шага к шагу и почему эта память затухает.",
+ "text": "Предложение приходит по одному слову, и рекуррентная ячейка переносит состояние h от шага к шагу: h' = tanh(W_x x + W_h h + b). В крошечной ячейке спеки W_x равен 1, а W_h 0.5, поэтому один импульс даёт 0.7616, 0.3634 и 0.1797: память о нём примерно вдвое слабеет каждый шаг. Второй импульс на шаге 3 поднимает h до 0.7968, чуть выше, чем начинался одиночный импульс, потому что старая память ещё там. После 8 шагов остаётся лишь 0.0466. Это затухание и есть причина, по которой простые ячейки забывают длинный контекст. Спека rnn_cell.t27 вычисляет каждый шаг.",
+ "task": "Загрузите один импульс и прочитайте, за сколько шагов он падает ниже 0.1; затем сделайте вход чередующимся."
+ }
+ },
+ {
+ "id": "lstm-gates",
+ "n": 23,
+ "module": "memory-and-attention",
+ "widget": {
+ "id": "nn-lstm",
+ "kind": "tool",
+ "title": "LSTM gates: how a cell decides what to keep",
+ "url": "https://t27.ai/widgets/nn-lstm/",
+ "preview": "widgets/nn-lstm/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-rnn",
+ "kind": "tool",
+ "title": "Recurrent cells: how a network remembers a moment ago",
+ "url": "https://t27.ai/widgets/nn-rnn/",
+ "preview": "widgets/nn-rnn/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-attention",
+ "kind": "tool",
+ "title": "Attention: how a word decides which other words to read",
+ "url": "https://t27.ai/widgets/nn-attention/",
+ "preview": "widgets/nn-attention/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-sigmoid",
+ "kind": "tool",
+ "title": "Sigmoid: turn any number into a probability",
+ "url": "https://t27.ai/widgets/nn-sigmoid/",
+ "preview": "widgets/nn-sigmoid/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/recurrent/lstm_single.t27",
+ "source": "t27/files/specs/ml/recurrent/lstm_single.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Frecurrent%2Flstm_single.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "LSTM gates",
+ "goal": "How the forget, input and output gates of an LSTM decide what the cell keeps, writes and shows.",
+ "text": "An LSTM cell keeps a separate memory c and three gates, each a sigmoid between 0 and 1: forget scales the old memory, input lets in new content g, and output decides how much of the memory to show. c' = f c + i g and h' = o tanh(c'). In the spec's fixture every weight is 0, so each gate is the sigmoid of its bias: with the forget gate at 0.9975 and the input gate at 0.0025, a memory of 1 comes out as 0.9975, and the output is 0.7587. Because c passes through with only a multiply by f, its gradient survives many steps too. The spec lstm_single.t27 computes the cell.",
+ "task": "Load Remember and then Forget and compare the new memory; then write with the input gate open."
+ },
+ "ru": {
+ "title": "Вентили LSTM",
+ "goal": "Как вентили забывания, входа и выхода LSTM решают, что ячейка хранит, записывает и показывает.",
+ "text": "Ячейка LSTM хранит отдельную память c и три вентиля, каждый это сигмоида между 0 и 1: забывание масштабирует старую память, вход впускает новое содержимое g, а выход решает, сколько памяти показать. c' = f c + i g и h' = o tanh(c'). В фикстуре спеки каждый вес равен 0, поэтому каждый вентиль это сигмоида своего смещения: при вентиле забывания 0.9975 и вентиле входа 0.0025 память 1 выходит как 0.9975, а выход равен 0.7587. Так как c проходит лишь с умножением на f, её градиент тоже переживает много шагов. Спека lstm_single.t27 вычисляет ячейку.",
+ "task": "Загрузите Remember, а затем Forget и сравните новую память; затем запишите с открытым вентилем входа."
+ }
+ },
+ {
+ "id": "attention",
+ "n": 24,
+ "module": "memory-and-attention",
+ "widget": {
+ "id": "nn-attention",
+ "kind": "tool",
+ "title": "Attention: how a word decides which other words to read",
+ "url": "https://t27.ai/widgets/nn-attention/",
+ "preview": "widgets/nn-attention/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-rnn",
+ "kind": "tool",
+ "title": "Recurrent cells: how a network remembers a moment ago",
+ "url": "https://t27.ai/widgets/nn-rnn/",
+ "preview": "widgets/nn-rnn/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-lstm",
+ "kind": "tool",
+ "title": "LSTM gates: how a cell decides what to keep",
+ "url": "https://t27.ai/widgets/nn-lstm/",
+ "preview": "widgets/nn-lstm/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-encoder",
+ "kind": "tool",
+ "title": "An encoder block: what one transformer layer does",
+ "url": "https://t27.ai/widgets/nn-encoder/",
+ "preview": "widgets/nn-encoder/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/recurrent/attention_mechanism.t27",
+ "source": "t27/files/specs/ml/recurrent/attention_mechanism.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Frecurrent%2Fattention_mechanism.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Attention",
+ "goal": "How attention scores every query against every key, what a softmax over each row gives, and what the causal mask removes.",
+ "text": "Attention lets each position decide which other positions to read. Every query is scored against every key, q . k / sqrt(d), and a softmax turns each row of scores into weights that sum to 1; the output is the values averaged by those weights. Two heads do it on the two halves of the vector. With a causal mask a position cannot read the future: in head 0, position 0 reads only itself, position 1 splits 0.452 and 0.548, and position 2 spreads 0.230, 0.321 and 0.449. Without the mask every row spreads over all three keys, from 0.21 to 0.47. The spec attention_mechanism.t27 computes every weight on its own fixture.",
+ "task": "Turn the mask off and on and read row 0; then compare the two heads on row 2."
+ },
+ "ru": {
+ "title": "Внимание",
+ "goal": "Как внимание сравнивает каждый запрос с каждым ключом, что даёт softmax по каждой строке и что убирает причинная маска.",
+ "text": "Внимание позволяет каждой позиции решить, какие другие позиции читать. Каждый запрос сравнивается с каждым ключом, q . k / sqrt(d), и softmax превращает каждую строку оценок в веса с суммой 1; выход это значения, усреднённые этими весами. Две головы делают это на двух половинах вектора. С причинной маской позиция не может читать будущее: в голове 0 позиция 0 читает только себя, позиция 1 делит 0.452 и 0.548, а позиция 2 распределяет 0.230, 0.321 и 0.449. Без маски каждая строка распределяется по всем трём ключам, от 0.21 до 0.47. Спека attention_mechanism.t27 вычисляет каждый вес на своей фикстуре.",
+ "task": "Выключите и включите маску и прочитайте строку 0; затем сравните две головы в строке 2."
+ }
+ },
+ {
+ "id": "positional-encoding",
+ "n": 25,
+ "module": "transformer-and-reward",
+ "widget": {
+ "id": "nn-positions",
+ "kind": "tool",
+ "title": "Positional encoding: how a transformer knows word order",
+ "url": "https://t27.ai/widgets/nn-positions/",
+ "preview": "widgets/nn-positions/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-attention",
+ "kind": "tool",
+ "title": "Attention: how a word decides which other words to read",
+ "url": "https://t27.ai/widgets/nn-attention/",
+ "preview": "widgets/nn-attention/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-encoder",
+ "kind": "tool",
+ "title": "An encoder block: what one transformer layer does",
+ "url": "https://t27.ai/widgets/nn-encoder/",
+ "preview": "widgets/nn-encoder/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-embedding",
+ "kind": "tool",
+ "title": "Embeddings: how a word becomes a row of numbers",
+ "url": "https://t27.ai/widgets/nn-embedding/",
+ "preview": "widgets/nn-embedding/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/transformer/positional_encoding.t27",
+ "source": "t27/files/specs/ml/transformer/positional_encoding.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Ftransformer%2Fpositional_encoding.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "Positional encoding",
+ "goal": "How sines and cosines at falling frequencies give every position its own vector, and why a shift is a turn.",
+ "text": "Attention treats its input as a set, so the order of the words has to be added to them. The sinusoidal encoding gives each position d numbers: the sin and cos of the position times a frequency that falls by 10000^(2/d) from pair to pair. With d = 8 the four pairs turn by 1, 0.1, 0.01 and 0.001 radians per position, like the hands of four clocks: position 5 reads -0.959, 0.284 in its fast pair and 0.005, 1.000 in its slow one. Each pair has length 1, and a shift by k positions turns every pair by its own fixed angle. The spec positional_encoding.t27 computes every value.",
+ "task": "Step from position 0 to 1 and watch which clock moves; then jump to 37 and find the pair that still points up."
+ },
+ "ru": {
+ "title": "Позиционное кодирование",
+ "goal": "Как синусы и косинусы убывающих частот дают каждой позиции свой вектор и почему сдвиг это поворот.",
+ "text": "Внимание видит свой вход как множество, поэтому порядок слов нужно к ним добавить. Синусоидальное кодирование даёт каждой позиции d чисел: sin и cos позиции, умноженной на частоту, которая от пары к паре падает в 10000^(2/d) раз. При d = 8 четыре пары поворачиваются на 1, 0.1, 0.01 и 0.001 радиана за позицию, как стрелки четырёх часов: позиция 5 читает -0.959, 0.284 в быстрой паре и 0.005, 1.000 в медленной. Длина каждой пары равна 1, и сдвиг на k позиций поворачивает каждую пару на свой постоянный угол. Спека positional_encoding.t27 вычисляет каждое значение.",
+ "task": "Шагните с позиции 0 на 1 и посмотрите, какие часы двигаются; затем перейдите на 37 и найдите пару, которая всё ещё смотрит вверх."
+ }
+ },
+ {
+ "id": "an-encoder-block",
+ "n": 26,
+ "module": "transformer-and-reward",
+ "widget": {
+ "id": "nn-encoder",
+ "kind": "tool",
+ "title": "An encoder block: what one transformer layer does",
+ "url": "https://t27.ai/widgets/nn-encoder/",
+ "preview": "widgets/nn-encoder/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-attention",
+ "kind": "tool",
+ "title": "Attention: how a word decides which other words to read",
+ "url": "https://t27.ai/widgets/nn-attention/",
+ "preview": "widgets/nn-attention/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-positions",
+ "kind": "tool",
+ "title": "Positional encoding: how a transformer knows word order",
+ "url": "https://t27.ai/widgets/nn-positions/",
+ "preview": "widgets/nn-positions/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-batchnorm",
+ "kind": "tool",
+ "title": "Batch norm: features on very different scales, made comparable",
+ "url": "https://t27.ai/widgets/nn-batchnorm/",
+ "preview": "widgets/nn-batchnorm/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/transformer/encoder_block.t27",
+ "source": "t27/files/specs/ml/transformer/encoder_block.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Ftransformer%2Fencoder_block.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "An encoder block",
+ "goal": "What one transformer encoder block does to its input, and what moving the norms in front changes.",
+ "text": "A transformer layer is two sublayers, each wrapped in a residual add and a layer norm: self-attention mixes the positions, then a feed-forward net of width 8 works on each position alone. In the post-norm order of the original paper the block ends with a norm, so every output row has mean 0 and variance 1.000. Pre-norm puts the norms in front of each sublayer and ends on a residual add, so its rows come out at variances 2.73, 3.33 and 0.96; deep stacks train more easily that way, with one norm added at the very top. The spec encoder_block.t27 runs one block on 3 positions, d_model 4 and 2 heads.",
+ "task": "Switch between post-norm and pre-norm and read the row variances; then find the position head 1 reads most."
+ },
+ "ru": {
+ "title": "Блок энкодера",
+ "goal": "Что один блок энкодера трансформера делает со своим входом и что меняет перенос нормализаций вперёд.",
+ "text": "Слой трансформера это два подслоя, каждый обёрнут остаточным сложением и нормализацией слоя: самовнимание смешивает позиции, затем сеть прямого распространения ширины 8 работает с каждой позицией отдельно. В порядке post-norm из исходной статьи блок кончается нормализацией, поэтому у каждой строки выхода среднее 0 и дисперсия 1.000. Pre-norm ставит нормализации перед каждым подслоем и кончается остаточным сложением, поэтому его строки выходят с дисперсиями 2.73, 3.33 и 0.96; глубокие стопки так учатся легче, с одной нормализацией в самом верху. Спека encoder_block.t27 прогоняет один блок на 3 позициях, d_model 4 и 2 головах.",
+ "task": "Переключайтесь между post-norm и pre-norm и читайте дисперсии строк; затем найдите позицию, которую голова 1 читает больше всего."
+ }
+ },
+ {
+ "id": "ppo-and-the-reward",
+ "n": 27,
+ "module": "transformer-and-reward",
+ "widget": {
+ "id": "nn-ppo",
+ "kind": "tool",
+ "title": "PPO: how an agent learns from reward without lurching",
+ "url": "https://t27.ai/widgets/nn-ppo/",
+ "preview": "widgets/nn-ppo/index.html?embed=1",
+ "height": 760
+ },
+ "also": [
+ {
+ "id": "nn-softmax",
+ "kind": "tool",
+ "title": "Softmax: how a network turns scores into a choice",
+ "url": "https://t27.ai/widgets/nn-softmax/",
+ "preview": "widgets/nn-softmax/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-adam",
+ "kind": "tool",
+ "title": "Adam: the optimizer most networks are trained with",
+ "url": "https://t27.ai/widgets/nn-adam/",
+ "preview": "widgets/nn-adam/index.html?embed=1",
+ "height": 760
+ },
+ {
+ "id": "nn-encoder",
+ "kind": "tool",
+ "title": "An encoder block: what one transformer layer does",
+ "url": "https://t27.ai/widgets/nn-encoder/",
+ "preview": "widgets/nn-encoder/index.html?embed=1",
+ "height": 760
+ }
+ ],
+ "spec": {
+ "path": "specs/ml/rl/ppo_clip_loss.t27",
+ "source": "t27/files/specs/ml/rl/ppo_clip_loss.t27",
+ "preview": "play/embed.html?spec=specs%2Fml%2Frl%2Fppo_clip_loss.t27&tab=term&v=7cbe9ab0c960",
+ "height": 480
+ },
+ "en": {
+ "title": "PPO and the reward",
+ "goal": "How PPO turns a reward into a policy update, and how its clip keeps one update from going too far.",
+ "text": "An agent learns from reward by making good actions more likely. PPO measures a change of policy by the ratio r of the new probability of an action to the old one and multiplies it by the advantage A, how much better the action did than expected. Then it clips: once r has moved more than eps the way A rewards, the objective goes flat. At r = 1.5, A = 1 and eps 0.2 the raw r A is 1.5, but PPO counts 1.2, so one batch cannot push the policy far. On the spec's batch, ratios 1, 1.1 and 0.9 with advantages 2, 1.5 and 3, the policy loss is -2.1167. The spec ppo_clip_loss.t27 computes it.",
+ "task": "Make the advantage negative and find where the curve goes flat now; then widen eps to 0.5."
+ },
+ "ru": {
+ "title": "PPO и награда",
+ "goal": "Как PPO превращает награду в обновление политики и как его обрезка не даёт одному обновлению зайти слишком далеко.",
+ "text": "Агент учится на награде, делая хорошие действия вероятнее. PPO измеряет изменение политики отношением r новой вероятности действия к старой и умножает его на преимущество A, насколько действие превзошло ожидание. Затем он обрезает: когда r ушло больше чем на eps в ту сторону, за которую награждает A, цель становится плоской. При r = 1.5, A = 1 и eps 0.2 сырое r A равно 1.5, но PPO засчитывает 1.2, так что один пакет не может далеко сдвинуть политику. На пакете спеки, отношения 1, 1.1 и 0.9 с преимуществами 2, 1.5 и 3, потеря политики равна -2.1167. Спека ppo_clip_loss.t27 её вычисляет.",
+ "task": "Сделайте преимущество отрицательным и найдите, где кривая теперь становится плоской; затем расширьте eps до 0.5."
+ }
+ }
+ ],
+ "prev": {
+ "id": "arithmetic-and-dsp",
+ "route": "arithmetic-and-dsp",
+ "share": "learn/arithmetic-and-dsp/",
+ "first": "the-carry-chain",
+ "last": "capstone-a-filter-on-the-board",
+ "title": {
+ "en": "Arithmetic and DSP on the FPGA: from the carry chain to a filter",
+ "ru": "Арифметика и DSP на FPGA: от цепи переноса до фильтра"
+ }
+ },
"next": null
}
] as const
diff --git a/apps/website/src/lib/courseRoutes.generated.ts b/apps/website/src/lib/courseRoutes.generated.ts
index ec8837b5c2..66a4753315 100644
--- a/apps/website/src/lib/courseRoutes.generated.ts
+++ b/apps/website/src/lib/courseRoutes.generated.ts
@@ -2,4 +2,4 @@
// Do not edit: change the .t27 source and regenerate.
/** Every course's address in the app (#/, #//), in reading order. */
-export const COURSE_ROUTES = ["t27-basics","course","ai-numbers","goldenfloat","verify-hardware","clocks-and-cdc","buses-and-peripherals","ternary-computing","arithmetic-and-dsp"] as const
+export const COURSE_ROUTES = ["t27-basics","course","ai-numbers","goldenfloat","verify-hardware","clocks-and-cdc","buses-and-peripherals","ternary-computing","arithmetic-and-dsp","neural-networks"] as const
diff --git a/apps/website/src/lib/queenWidgets.generated.ts b/apps/website/src/lib/queenWidgets.generated.ts
index e3cfbfac81..ec049a5412 100644
--- a/apps/website/src/lib/queenWidgets.generated.ts
+++ b/apps/website/src/lib/queenWidgets.generated.ts
@@ -1,12 +1,12 @@
// GENERATED by scripts/widgets-from-spec.mjs from specs/widgets/gallery.t27
-// spec sha256 0e72eea319b09e2a1d9ad2a4875f01334baf8cb056a64461be511ecf93df8cb1
+// spec sha256 1f89127fb05172d18af3b035147793a5729c85ef722609555e2e7de68d43b7eb
// Do not edit: change the .t27 source and regenerate.
export const QUEEN_WIDGETS = {
"source": {
"spec": "specs/widgets/gallery.t27",
"publicSpec": "public/widgets/gallery.t27",
- "sha256": "0e72eea319b09e2a1d9ad2a4875f01334baf8cb056a64461be511ecf93df8cb1",
+ "sha256": "1f89127fb05172d18af3b035147793a5729c85ef722609555e2e7de68d43b7eb",
"schemaVersion": 1
},
"name": "t27 widgets",
@@ -4555,6 +4555,465 @@ export const QUEEN_WIDGETS = {
"iframe": "",
"markdown": "[](https://t27.ai/widgets/tc-tiny-layer/)"
},
+ {
+ "id": "nn-relu",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "max(0, x) is the whole rule, and its slope is 1 or 0. A leaky slope of 0.1 keeps a gradient alive on the negative side.",
+ "title": "ReLU: what a neuron does with a negative number",
+ "line": "Drag along the curve and watch ReLU pass every positive input and zero every negative one; give it a leaky slope and a little of the negative side gets through.",
+ "url": "https://t27.ai/widgets/nn-relu/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-relu/card.png?v=d15fbb878508",
+ "gif": null,
+ "preview": "widgets/nn-relu/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-relu/)"
+ },
+ {
+ "id": "nn-sigmoid",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "1 / (1 + e^-x) squeezes the whole line into 0..1. Its slope is never above 0.25, which is why deep sigmoid nets learned slowly.",
+ "title": "Sigmoid: turn any number into a probability",
+ "line": "Drag along the S-curve: every input lands between 0 and 1, and the slope peaks at 0.25 in the middle and dies away at both ends.",
+ "url": "https://t27.ai/widgets/nn-sigmoid/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-sigmoid/card.png?v=cfbac761b571",
+ "gif": null,
+ "preview": "widgets/nn-sigmoid/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-sigmoid/)"
+ },
+ {
+ "id": "nn-tanh",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "tanh(2.5) is already 0.987 and its slope 0.027. Past about 1.8 either way the gradient is under 0.1: the band a deep net must stay out of.",
+ "title": "tanh: where a neuron stops learning",
+ "line": "Drag along tanh: it is centred on zero, and where it flattens out its slope drops under 0.1. The red band shows every input where the gradient has all but stopped.",
+ "url": "https://t27.ai/widgets/nn-tanh/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-tanh/card.png?v=79db6b823fa6",
+ "gif": null,
+ "preview": "widgets/nn-tanh/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-tanh/)"
+ },
+ {
+ "id": "nn-gelu",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "GELU(-0.75) = -0.16997 exactly and -0.17004 by the tanh form. A smooth ReLU with a small dip, used in BERT and GPT.",
+ "title": "GELU: why transformers bend a little below zero",
+ "line": "Drag along GELU: x weighed by the chance a normal draw is below it. It dips to about -0.17 before it rises, and the tanh shortcut stays within a few thousandths of the exact form.",
+ "url": "https://t27.ai/widgets/nn-gelu/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-gelu/card.png?v=c2b42ff605ab",
+ "gif": null,
+ "preview": "widgets/nn-gelu/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-gelu/)"
+ },
+ {
+ "id": "nn-silu",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "SiLU(-1.28) = -0.2785, its lowest point. At beta 8 the dip shrinks to almost nothing and the curve is ReLU with rounded corners.",
+ "title": "Swish: one knob between a smooth curve and ReLU",
+ "line": "Drag along x times sigmoid(beta x). At beta 1 it is SiLU, with its dip of -0.28; turn beta up and the curve closes on ReLU, turn it down and it flattens toward x / 2.",
+ "url": "https://t27.ai/widgets/nn-silu/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-silu/card.png?v=d21993bc6b54",
+ "gif": null,
+ "preview": "widgets/nn-silu/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-silu/)"
+ },
+ {
+ "id": "nn-elu",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "ELU(-3) with alpha 1 is -0.9502, already 95% of the way to its floor of -1, and its slope there is still 0.0498, not 0.",
+ "title": "ELU: a negative side with a floor",
+ "line": "Drag along ELU: x above zero, alpha (e^x - 1) below. The negative side never falls under -alpha, so outputs average closer to zero, and its slope never quite dies.",
+ "url": "https://t27.ai/widgets/nn-elu/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-elu/card.png?v=68396c794178",
+ "gif": null,
+ "preview": "widgets/nn-elu/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-elu/)"
+ },
+ {
+ "id": "nn-softmax",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Scores 1, 2, 3 give 0.090, 0.245, 0.665. At temperature 0.5 the top one takes 0.867; at 5 the three are 0.27, 0.33, 0.40.",
+ "title": "Softmax: how a network turns scores into a choice",
+ "line": "Move three scores and the temperature: softmax turns them into probabilities that sum to one. Cool it and the top score takes almost everything; heat it and the choice evens out.",
+ "url": "https://t27.ai/widgets/nn-softmax/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-softmax/card.png?v=694f0541a6a9",
+ "gif": null,
+ "preview": "widgets/nn-softmax/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-softmax/)"
+ },
+ {
+ "id": "nn-embedding",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "A token is an index into a table, nothing more. Read token 2 twice and its row's gradient is (1 + 3, 2 + 4) = (4, 6); the other rows get 0.",
+ "title": "Embeddings: how a word becomes a row of numbers",
+ "line": "Pick two tokens: each one copies its row out of the table. Then the gradient flows back only into the rows that were read, and a token read twice collects both.",
+ "url": "https://t27.ai/widgets/nn-embedding/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-embedding/card.png?v=8fe367521243",
+ "gif": null,
+ "preview": "widgets/nn-embedding/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-embedding/)"
+ },
+ {
+ "id": "nn-dropout",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "With p = 0.5, inverted dropout doubles the units it keeps while training (1, 2, 3, 4 becomes 2, 0, 6, 0) and leaves inference alone.",
+ "title": "Dropout: training a network with neurons switched off",
+ "line": "Switch between training and inference, inverted and classic dropout: units 1 and 3 are dropped, the survivors are scaled so that the layer's output keeps its expected size.",
+ "url": "https://t27.ai/widgets/nn-dropout/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-dropout/card.png?v=8a4ce725876d",
+ "gif": null,
+ "preview": "widgets/nn-dropout/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-dropout/)"
+ },
+ {
+ "id": "nn-conv",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Pictures are read in patches. Here a 2 x 2 window steps by 2 over a 4 x 4 image and averages: 2.5, 4.5, 10.5, 12.5. A learned kernel comes next.",
+ "title": "Patches of a picture: a 2 x 2 window, averaged",
+ "line": "Tap a window on the 4 x 4 image: each output is the average of its 2 x 2 patch, two cells apart. Then switch to the gradient and see each output's blame shared out over its patch.",
+ "url": "https://t27.ai/widgets/nn-conv/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-conv/card.png?v=b42c26e28e28",
+ "gif": null,
+ "preview": "widgets/nn-conv/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-conv/)"
+ },
+ {
+ "id": "nn-maxpool",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "On the spec's 4 x 4 grid, 2 x 2 windows with stride 2 keep 4, 8, 9 and -2; their gradients land on cells 5, 6, 13 and 11 and nowhere else.",
+ "title": "Max pooling: what a network keeps when it shrinks a picture",
+ "line": "Pick a window size and stride and tap an output: max pooling keeps the largest value of its window and remembers where it was, and the gradient later goes to that one cell.",
+ "url": "https://t27.ai/widgets/nn-maxpool/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-maxpool/card.png?v=e9fe38153d4b",
+ "gif": null,
+ "preview": "widgets/nn-maxpool/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-maxpool/)"
+ },
+ {
+ "id": "nn-batchnorm",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Features 1..4 and 10..40 come out identical, -1.34, -0.45, 0.45, 1.34: batch norm keeps the shape of each feature and throws away its scale.",
+ "title": "Batch norm: features on very different scales, made comparable",
+ "line": "A batch of 4 samples whose 3 features live on scales 1, 10 and 1: batch norm gives each feature mean 0 and variance 1. Then set gamma and the running statistics for inference.",
+ "url": "https://t27.ai/widgets/nn-batchnorm/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-batchnorm/card.png?v=859705989349",
+ "gif": null,
+ "preview": "widgets/nn-batchnorm/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-batchnorm/)"
+ },
+ {
+ "id": "nn-mse",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "A miss of 1 costs 1 / 3 of the mean; a miss of 2 costs 4 / 3. Squaring is what makes MSE fear large errors and shrug at small ones.",
+ "title": "Mean squared error: how wrong a guess is, as one number",
+ "line": "Move three predictions against the targets 1, 2 and 3: the loss is the mean of the squared misses, its bowl is a parabola, and each gradient points its prediction back at the target.",
+ "url": "https://t27.ai/widgets/nn-mse/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-mse/card.png?v=a23aaeb501dc",
+ "gif": null,
+ "preview": "widgets/nn-mse/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-mse/)"
+ },
+ {
+ "id": "nn-bce",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Predicting 0.99 where the answer is 1 costs 0.01; predicting 0.01 costs 4.6. The cost of a confident mistake is what trains a classifier.",
+ "title": "Cross-entropy: why a confident wrong answer costs so much",
+ "line": "Set three predicted probabilities against the answers 1, 0, 1: the loss is -log of the chance given to the truth. Drag toward a confident mistake and the cost climbs without bound.",
+ "url": "https://t27.ai/widgets/nn-bce/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-bce/card.png?v=8f6557e0c831",
+ "gif": null,
+ "preview": "widgets/nn-bce/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-bce/)"
+ },
+ {
+ "id": "nn-huber",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "An error of 3 costs 2.5 under Huber with delta 1 and 4.5 squared. At 30 the gap is 29.5 against 450: why robust regression uses it.",
+ "title": "Huber loss: a loss that does not panic at an outlier",
+ "line": "Drag the error along: Huber is half its square inside delta and only grows linearly outside, so one wild miss pulls on the model with a bounded force. The dashed curve is the pure square.",
+ "url": "https://t27.ai/widgets/nn-huber/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-huber/card.png?v=9e8ba97cf835",
+ "gif": null,
+ "preview": "widgets/nn-huber/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-huber/)"
+ },
+ {
+ "id": "nn-sgd",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "lr 0.1 reaches 2.99996 in 50 steps. Add momentum 0.9 and it shoots to 5.13 first; at lr 1 it bounces between 0 and 6 for good.",
+ "title": "Gradient descent: how a network walks downhill",
+ "line": "Set the learning rate and momentum and watch a weight walk from 0 to the bottom of (x - 3)^2. Plain steps creep, momentum overshoots and rings, and a rate of 1 bounces forever.",
+ "url": "https://t27.ai/widgets/nn-sgd/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-sgd/card.png?v=09aa5d22c974",
+ "gif": null,
+ "preview": "widgets/nn-sgd/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-sgd/)"
+ },
+ {
+ "id": "nn-rmsprop",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Its first step is 0.0316 whether the gradient is 0.001 or 1000. Dividing by the gradient's own size is what makes the step scale-free.",
+ "title": "RMSProp: a step that does not care how big the gradient is",
+ "line": "Feed one weight a constant gradient of 0.001, 0.5 or 1000: RMSProp divides by the root of its running mean square, so all three walk the same path. Change the decay and the path bends.",
+ "url": "https://t27.ai/widgets/nn-rmsprop/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-rmsprop/card.png?v=8976db28abff",
+ "gif": null,
+ "preview": "widgets/nn-rmsprop/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-rmsprop/)"
+ },
+ {
+ "id": "nn-adam",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Gradients 2, -40 and 0.5 all move their weights by exactly 0.001 on step 1. Adam reaches 3.00005 in 200 steps at lr 0.1.",
+ "title": "Adam: the optimizer most networks are trained with",
+ "line": "Watch Adam carry a weight from 0 to the minimum at 3, then set three gradients of any size: the bias correction makes the first step lr times their sign, whatever their size.",
+ "url": "https://t27.ai/widgets/nn-adam/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-adam/card.png?v=8fad2efe1102",
+ "gif": null,
+ "preview": "widgets/nn-adam/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-adam/)"
+ },
+ {
+ "id": "nn-backprop",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "With g = (1, 0.5) on inputs (1, 2, 3), the weights get 1, 2, 3 and 0.5, 1, 1.5, and the inputs get 0.5, 2, 3.5. Two products are the whole rule.",
+ "title": "Backpropagation: how blame flows back through a layer",
+ "line": "Set how much the loss wants each of two outputs to move: every weight gets that times its input, every input gets W transposed times it. The edges and nodes show the numbers.",
+ "url": "https://t27.ai/widgets/nn-backprop/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-backprop/card.png?v=a8388b67dc9e",
+ "gif": null,
+ "preview": "widgets/nn-backprop/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-backprop/)"
+ },
+ {
+ "id": "nn-xor",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "One neuron cannot draw XOR. Eight hidden units can: after 2000 epochs the outputs are 0, 1, 1, 0 to within 4e-7 and the loss is 1.8e-13.",
+ "title": "XOR: a network learns what one neuron cannot",
+ "line": "A 2-8-1 network trained by backpropagation in your browser: drag the epochs from 0 to 2000 and watch its four answers move to 0, 1, 1, 0 while the loss falls from 0.89 by twelve orders.",
+ "url": "https://t27.ai/widgets/nn-xor/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-xor/card.png?v=78f8c0793da2",
+ "gif": null,
+ "preview": "widgets/nn-xor/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-xor/)"
+ },
+ {
+ "id": "nn-lamb",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Weights (2, 4) take a step of 0.00447 and weights (6, 12) one of 0.01: a step is 0.001 of the layer's norm, capped at 10. That is how LAMB trains large batches.",
+ "title": "LAMB: one learning rate for layers of every size",
+ "line": "Scale a layer's weights and watch its LAMB step: it grows with the weight norm, so every layer moves by the same fraction of itself, until the norm passes 10 and the step stops growing.",
+ "url": "https://t27.ai/widgets/nn-lamb/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-lamb/card.png?v=2ef9515d8b3c",
+ "gif": null,
+ "preview": "widgets/nn-lamb/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-lamb/)"
+ },
+ {
+ "id": "nn-rnn",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "A single pulse gives 0.762, 0.363, 0.180: the memory halves each step. A cell remembers, but not for long, which is the problem LSTMs solve.",
+ "title": "Recurrent cells: how a network remembers a moment ago",
+ "line": "Tap the inputs of eight steps: the cell feeds its own last output back, h' = tanh(x + 0.5 h), so every pulse echoes on and fades by about half a step. Two pulses add up.",
+ "url": "https://t27.ai/widgets/nn-rnn/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-rnn/card.png?v=0c5ff53a846f",
+ "gif": null,
+ "preview": "widgets/nn-rnn/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-rnn/)"
+ },
+ {
+ "id": "nn-lstm",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "With the forget gate open (0.9975) and the input gate shut, a memory of 1 comes out as 0.9975: an LSTM can hold a value for a long time.",
+ "title": "LSTM gates: how a cell decides what to keep",
+ "line": "Set the biases of the forget, input and output gates and the old memory: the new memory is forget x old + input x candidate, and the output shows tanh of it through the output gate.",
+ "url": "https://t27.ai/widgets/nn-lstm/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-lstm/card.png?v=72b9d9347693",
+ "gif": null,
+ "preview": "widgets/nn-lstm/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-lstm/)"
+ },
+ {
+ "id": "nn-attention",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Causal: position 0 reads only itself (1.000), position 1 splits 0.45 / 0.55, position 2 spreads 0.23, 0.32, 0.45. Every row is a softmax.",
+ "title": "Attention: how a word decides which other words to read",
+ "line": "Two heads of attention over three positions: each row is how much one query reads each key. Switch on the causal mask and a position can no longer read the future.",
+ "url": "https://t27.ai/widgets/nn-attention/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-attention/card.png?v=8a8abc72d040",
+ "gif": null,
+ "preview": "widgets/nn-attention/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-attention/)"
+ },
+ {
+ "id": "nn-positions",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Position 5 is (-0.959, 0.284, 0.479, 0.878, 0.050, 0.999, 0.005, 1.000). Four clocks at speeds 1, 0.1, 0.01 and 0.001 give each position its own reading.",
+ "title": "Positional encoding: how a transformer knows word order",
+ "line": "Every position gets 8 numbers, sines and cosines at frequencies falling by 10 per pair. Drag the position: the fast clock spins, the slow ones barely move, and the stripes show all 48 rows.",
+ "url": "https://t27.ai/widgets/nn-positions/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-positions/card.png?v=1a3bfc4fdceb",
+ "gif": null,
+ "preview": "widgets/nn-positions/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-positions/)"
+ },
+ {
+ "id": "nn-encoder",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "Post-norm rows come out at mean 0 and variance 1.000; pre-norm rows at variances 2.73, 3.33 and 0.96, because its last step is a residual add.",
+ "title": "An encoder block: what one transformer layer does",
+ "line": "Three positions through one transformer block: attention, a residual, a feed-forward net and layer norm. Post-norm ends every row at mean 0 and variance 1; pre-norm does not.",
+ "url": "https://t27.ai/widgets/nn-encoder/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-encoder/card.png?v=ffd2cfddeda0",
+ "gif": null,
+ "preview": "widgets/nn-encoder/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-encoder/)"
+ },
+ {
+ "id": "nn-ppo",
+ "kind": "tool",
+ "category": "compiler",
+ "hook": "At r = 1.5 with advantage 1 and eps 0.2, the raw objective is 1.5 but PPO counts only 1.2. The clip is what keeps one update from breaking the policy.",
+ "title": "PPO: how an agent learns from reward without lurching",
+ "line": "Drag the probability ratio of a new policy over the old: PPO's objective follows r times the advantage until r moves more than eps the rewarded way, then it goes flat and stops pulling.",
+ "url": "https://t27.ai/widgets/nn-ppo/",
+ "view": null,
+ "image": "https://t27.ai/widgets/nn-ppo/card.png?v=432b62b0ecd7",
+ "gif": null,
+ "preview": "widgets/nn-ppo/index.html?embed=1",
+ "commands": [],
+ "recorded": null,
+ "iframe": "",
+ "markdown": "[](https://t27.ai/widgets/nn-ppo/)"
+ },
{
"id": "wars",
"kind": "tab",
Trinity S³AI