diff --git a/dtable_events/app/config.py b/dtable_events/app/config.py index fbd2e151..ba322fe7 100644 --- a/dtable_events/app/config.py +++ b/dtable_events/app/config.py @@ -20,6 +20,7 @@ def get_llm_prices(models): - ... price: input_tokens: float # / (M Tokens) + cached_tokens: float # / (M Tokens), defaults to input_tokens output_tokens: float # / (M Tokens) Type B: for embeddings or other models @@ -41,12 +42,14 @@ def get_llm_prices(models): if isinstance(price, (float, int)): prices[model_name] = { 'input_tokens': price, + 'cached_tokens': price, 'output_tokens': 0 } continue if isinstance(price, dict) and 'input_tokens' in price: prices[model_name] = { 'input_tokens': price.get('input_tokens', 0), + 'cached_tokens': price.get('cached_tokens', price.get('input_tokens', 0)), 'output_tokens': price.get('output_tokens', 0), } return prices diff --git a/dtable_events/tasks/ai_stats_worker.py b/dtable_events/tasks/ai_stats_worker.py index 6420cfb7..661ede74 100644 --- a/dtable_events/tasks/ai_stats_worker.py +++ b/dtable_events/tasks/ai_stats_worker.py @@ -18,6 +18,18 @@ logger = logging.getLogger(__name__) +def calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens): + prices = AI_PRICES[model] + input_price = prices.get('input_tokens') or 0 + cached_price = prices.get('cached_tokens', input_price) + output_price = prices.get('output_tokens') or 0 + return ( + input_price * ((input_tokens - cached_tokens) / 1000000) + + cached_price * (cached_tokens / 1000000) + + output_price * (output_tokens / 1000000) + ) + + class AIStatsWorker(object): def __init__(self): @@ -36,9 +48,9 @@ def _parse_config(self): self._enabled = AI_STATS_ENABLED def reset_stats(self): - self.org_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) - self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) - self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) + self.org_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'cached_tokens': 0, 'output_tokens': 0})) + self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'cached_tokens': 0, 'output_tokens': 0})) + self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'cached_tokens': 0, 'output_tokens': 0})) self.dtable_org_cache = {} self.dtable_workspace_owner_cache = {} @@ -102,6 +114,10 @@ def save_to_memory(self, usage_info, session): usage['input_tokens'] = 0 if not isinstance(usage.get('output_tokens'), int): usage['output_tokens'] = 0 + cached_tokens = usage.get('cached_tokens') or 0 + if not isinstance(cached_tokens, int): + cached_tokens = 0 + cached_tokens = min(max(cached_tokens, 0), max(usage['input_tokens'], 0)) owner_info = self._get_dtable_owner_info(dtable_uuid, session, org_id=org_id) if not owner_info: @@ -115,11 +131,14 @@ def save_to_memory(self, usage_info, session): return self.org_stats[org_id][model]['input_tokens'] += usage.get('input_tokens') or 0 + self.org_stats[org_id][model]['cached_tokens'] += cached_tokens self.org_stats[org_id][model]['output_tokens'] += usage.get('output_tokens') or 0 self.owner_stats[owner][model]['input_tokens'] += usage.get('input_tokens') or 0 + self.owner_stats[owner][model]['cached_tokens'] += cached_tokens self.owner_stats[owner][model]['output_tokens'] += usage.get('output_tokens') or 0 key = (model, scenario) self.dtable_stats[dtable_uuid][key]['input_tokens'] += usage.get('input_tokens') or 0 + self.dtable_stats[dtable_uuid][key]['cached_tokens'] += cached_tokens self.dtable_stats[dtable_uuid][key]['output_tokens'] += usage.get('output_tokens') or 0 def receive(self): @@ -197,27 +216,26 @@ def stats_worker(self): team_data = [] team_sql = ''' - INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:org_id, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `input_tokens`, `cached_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:org_id, :month, :model, :input_tokens, :cached_tokens, :output_tokens, :cost, :created_at, :updated_at) ''' for org_id, models_dict in org_stats.items(): for model, usage in models_dict.items(): input_tokens = usage.get('input_tokens') or 0 + cached_tokens = usage.get('cached_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 - input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 - output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 - input_cost = input_tokens_price * (input_tokens / 1000000) - output_cost = output_tokens_price * (output_tokens / 1000000) - logger.info('org %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', org_id, model, input_tokens, input_cost, output_tokens, output_cost) + cost = calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens) + logger.info('org %s model %s, input_tokens %s, cached_tokens %s, output_tokens %s, cost %s', org_id, model, input_tokens, cached_tokens, output_tokens, cost) params = { 'org_id': org_id, 'month': month, 'model': model, 'input_tokens': input_tokens, + 'cached_tokens': cached_tokens, 'output_tokens': output_tokens, - 'cost': input_cost + output_cost, + 'cost': cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } @@ -225,27 +243,26 @@ def stats_worker(self): owner_data = [] owner_sql = ''' - INSERT INTO `stats_ai_by_owner`(`owner`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:owner, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + INSERT INTO `stats_ai_by_owner`(`owner`, `month`, `model`, `input_tokens`, `cached_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:owner, :month, :model, :input_tokens, :cached_tokens, :output_tokens, :cost, :created_at, :updated_at) ''' for owner_id, models_dict in owner_stats.items(): for model, usage in models_dict.items(): input_tokens = usage.get('input_tokens') or 0 + cached_tokens = usage.get('cached_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 - input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 - output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 - input_cost = input_tokens_price * (input_tokens / 1000000) - output_cost = output_tokens_price * (output_tokens / 1000000) - logger.info('owner %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', owner_id, model, input_tokens, input_cost, output_tokens, output_cost) + cost = calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens) + logger.info('owner %s model %s, input_tokens %s, cached_tokens %s, output_tokens %s, cost %s', owner_id, model, input_tokens, cached_tokens, output_tokens, cost) params = { 'owner': owner_id, 'month': month, 'model': model, 'input_tokens': input_tokens, + 'cached_tokens': cached_tokens, 'output_tokens': output_tokens, - 'cost': input_cost + output_cost, + 'cost': cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } @@ -253,9 +270,10 @@ def stats_worker(self): dtable_data = [] dtable_sql = ''' - INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `group_id`, `org_id`, `scenario`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:dtable_uuid, :date, :model, :owner, :group_id, :org_id, :scenario, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `group_id`, `org_id`, `scenario`, `input_tokens`, `cached_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:dtable_uuid, :date, :model, :owner, :group_id, :org_id, :scenario, :input_tokens, :cached_tokens, :output_tokens, :cost, :created_at, :updated_at) ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`), + `cached_tokens`=`cached_tokens`+VALUES(`cached_tokens`), `output_tokens`=`output_tokens`+VALUES(`output_tokens`), `cost`=`cost`+VALUES(`cost`), `updated_at`=VALUES(`updated_at`) @@ -281,13 +299,11 @@ def stats_worker(self): for (model, scenario), usage in models_dict.items(): input_tokens = usage.get('input_tokens') or 0 + cached_tokens = usage.get('cached_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 - input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 - output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 - input_cost = input_tokens_price * (input_tokens / 1000000) - output_cost = output_tokens_price * (output_tokens / 1000000) - logger.info('dtable %s model %s scenario %s, input_tokens %s cost %s, output_tokens %s cost %s', dtable_uuid, model, scenario, input_tokens, input_cost, output_tokens, output_cost) + cost = calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens) + logger.info('dtable %s model %s scenario %s, input_tokens %s, cached_tokens %s, output_tokens %s, cost %s', dtable_uuid, model, scenario, input_tokens, cached_tokens, output_tokens, cost) params = { 'dtable_uuid': dtable_uuid, @@ -298,8 +314,9 @@ def stats_worker(self): 'org_id': org_id, 'scenario': scenario, 'input_tokens': input_tokens, + 'cached_tokens': cached_tokens, 'output_tokens': output_tokens, - 'cost': input_cost + output_cost, + 'cost': cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } diff --git a/tests/test_cached_tokens.py b/tests/test_cached_tokens.py new file mode 100644 index 00000000..8b42dc72 --- /dev/null +++ b/tests/test_cached_tokens.py @@ -0,0 +1,93 @@ +"""Exercise statistics without initializing deployment config, Redis or MySQL.""" +import ast +from collections import defaultdict +from copy import deepcopy +from datetime import datetime +import logging +from pathlib import Path +import unittest +from unittest.mock import MagicMock + + +SOURCE = Path(__file__).resolve().parents[1] / 'dtable_events' + + +def load_definitions(path, names, namespace): + tree = ast.parse(path.read_text()) + tree.body = [node for node in tree.body if isinstance(node, (ast.ClassDef, ast.FunctionDef)) and node.name in names] + exec(compile(tree, str(path), 'exec'), namespace) + return namespace + + +class CachedTokensTest(unittest.TestCase): + def setUp(self): + self.prices = {'test-model': {'input_tokens': 2, 'cached_tokens': 0.5, 'output_tokens': 4}} + self.namespace = load_definitions(SOURCE / 'tasks/ai_stats_worker.py', + {'AIStatsWorker', 'calculate_usage_cost'}, { + 'AI_PRICES': self.prices, + 'defaultdict': defaultdict, + 'deepcopy': deepcopy, + 'datetime': datetime, + 'logger': logging.getLogger(__name__), + 'text': lambda sql: sql, + 'uuid_str_to_32_chars': lambda uuid: uuid.replace('-', ''), + }) + worker_class = self.namespace['AIStatsWorker'] + self.worker = worker_class.__new__(worker_class) + self.worker.reset_stats() + self.worker._get_dtable_owner_info = MagicMock(return_value={'owner': 'owner', 'org_id': 1}) + + def save(self, cached_tokens=None, canonical=False): + usage = {'input_tokens': 1000, 'output_tokens': 100} if canonical else {'prompt_tokens': 1000, 'completion_tokens': 100} + if cached_tokens is not None: + usage['cached_tokens'] = cached_tokens + self.worker.save_to_memory({ + 'model': 'test-model', 'dtable_uuid': 'base', 'usage': usage, + 'scenario': 'app_page_generation', + }, MagicMock()) + + def test_normalize_and_aggregate(self): + for value, expected in [(None, 0), (0, 0), (400, 400), (1000, 1000), (-10, 0), (2000, 1000), ('bad', 0)]: + with self.subTest(cached_tokens=value): + self.worker.reset_stats() + self.save(value) + self.save(value, canonical=True) + expected_usage = {'input_tokens': 2000, 'cached_tokens': expected * 2, 'output_tokens': 200} + self.assertEqual(self.worker.org_stats[1]['test-model'], expected_usage) + self.assertEqual(self.worker.owner_stats['owner']['test-model'], expected_usage) + self.assertEqual(self.worker.dtable_stats['base'][('test-model', 'app_page_generation')], expected_usage) + + def test_cost_and_default_price(self): + calculate = self.namespace['calculate_usage_cost'] + for cached, expected in [(0, 0.0024), (400, 0.0018), (1000, 0.0009)]: + self.assertAlmostEqual(calculate('test-model', 1000, cached, 100), expected) + self.prices['test-model']['cached_tokens'] = 0 + self.assertAlmostEqual(calculate('test-model', 1000, 400, 100), 0.0016) + del self.prices['test-model']['cached_tokens'] + self.assertAlmostEqual(calculate('test-model', 1000, 400, 100), 0.0024) + + def test_flush_all_three_statistics_tables(self): + self.save(400) + self.worker.stats_lock = MagicMock() + self.worker.query_dtable_owners = MagicMock(return_value={'base': {'owner': 'owner', 'org_id': 1}}) + session = MagicMock() + self.worker._db_session_class = MagicMock(return_value=session) + self.worker.stats_worker() + self.assertEqual(session.execute.call_count, 3) + for call in session.execute.call_args_list: + sql, rows = call.args + self.assertIn('`cached_tokens`', sql) + self.assertEqual(rows[0]['cached_tokens'], 400) + self.assertAlmostEqual(rows[0]['cost'], 0.0018) + session.commit.assert_called_once() + + def test_price_configuration(self): + parse = load_definitions(SOURCE / 'app/config.py', {'get_llm_prices'}, {})['get_llm_prices'] + for price, expected in [(2, 2), ({'input_tokens': 2}, 2), ({'input_tokens': 2, 'cached_tokens': 0}, 0), + ({'input_tokens': 2, 'cached_tokens': 0.5}, 0.5)]: + with self.subTest(price=price): + self.assertEqual(parse([{'model': 'test-model', 'price': price}])['test-model']['cached_tokens'], expected) + + +if __name__ == '__main__': + unittest.main()