Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dtable_events/app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ def get_llm_prices(models):
- ...
price:
input_tokens: float # / (M Tokens)
cached_tokens: float # / (M Tokens), defaults to input_tokens
output_tokens: float # / (M Tokens)

Type B: for embeddings or other models
Expand All @@ -41,12 +42,14 @@ def get_llm_prices(models):
if isinstance(price, (float, int)):
prices[model_name] = {
'input_tokens': price,
'cached_tokens': price,
'output_tokens': 0
}
continue
if isinstance(price, dict) and 'input_tokens' in price:
prices[model_name] = {
'input_tokens': price.get('input_tokens', 0),
'cached_tokens': price.get('cached_tokens', price.get('input_tokens', 0)),
'output_tokens': price.get('output_tokens', 0),
}
return prices
Expand Down
71 changes: 44 additions & 27 deletions dtable_events/tasks/ai_stats_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,18 @@
logger = logging.getLogger(__name__)


def calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens):
prices = AI_PRICES[model]
input_price = prices.get('input_tokens') or 0
cached_price = prices.get('cached_tokens', input_price)
output_price = prices.get('output_tokens') or 0
return (
input_price * ((input_tokens - cached_tokens) / 1000000) +
cached_price * (cached_tokens / 1000000) +
output_price * (output_tokens / 1000000)
)


class AIStatsWorker(object):

def __init__(self):
Expand All @@ -36,9 +48,9 @@ def _parse_config(self):
self._enabled = AI_STATS_ENABLED

def reset_stats(self):
self.org_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0}))
self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0}))
self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0}))
self.org_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'cached_tokens': 0, 'output_tokens': 0}))
self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'cached_tokens': 0, 'output_tokens': 0}))
self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'cached_tokens': 0, 'output_tokens': 0}))
self.dtable_org_cache = {}
self.dtable_workspace_owner_cache = {}

Expand Down Expand Up @@ -102,6 +114,10 @@ def save_to_memory(self, usage_info, session):
usage['input_tokens'] = 0
if not isinstance(usage.get('output_tokens'), int):
usage['output_tokens'] = 0
cached_tokens = usage.get('cached_tokens') or 0
if not isinstance(cached_tokens, int):
cached_tokens = 0
cached_tokens = min(max(cached_tokens, 0), max(usage['input_tokens'], 0))

owner_info = self._get_dtable_owner_info(dtable_uuid, session, org_id=org_id)
if not owner_info:
Expand All @@ -115,11 +131,14 @@ def save_to_memory(self, usage_info, session):
return

self.org_stats[org_id][model]['input_tokens'] += usage.get('input_tokens') or 0
self.org_stats[org_id][model]['cached_tokens'] += cached_tokens
self.org_stats[org_id][model]['output_tokens'] += usage.get('output_tokens') or 0
self.owner_stats[owner][model]['input_tokens'] += usage.get('input_tokens') or 0
self.owner_stats[owner][model]['cached_tokens'] += cached_tokens
self.owner_stats[owner][model]['output_tokens'] += usage.get('output_tokens') or 0
key = (model, scenario)
self.dtable_stats[dtable_uuid][key]['input_tokens'] += usage.get('input_tokens') or 0
self.dtable_stats[dtable_uuid][key]['cached_tokens'] += cached_tokens
self.dtable_stats[dtable_uuid][key]['output_tokens'] += usage.get('output_tokens') or 0

def receive(self):
Expand Down Expand Up @@ -197,65 +216,64 @@ def stats_worker(self):

team_data = []
team_sql = '''
INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`)
VALUES (:org_id, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at)
INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `input_tokens`, `cached_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`)
VALUES (:org_id, :month, :model, :input_tokens, :cached_tokens, :output_tokens, :cost, :created_at, :updated_at)
'''
for org_id, models_dict in org_stats.items():
for model, usage in models_dict.items():
input_tokens = usage.get('input_tokens') or 0
cached_tokens = usage.get('cached_tokens') or 0
output_tokens = usage.get('output_tokens') or 0

input_tokens_price = AI_PRICES[model].get('input_tokens') or 0
output_tokens_price = AI_PRICES[model].get('output_tokens') or 0
input_cost = input_tokens_price * (input_tokens / 1000000)
output_cost = output_tokens_price * (output_tokens / 1000000)
logger.info('org %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', org_id, model, input_tokens, input_cost, output_tokens, output_cost)
cost = calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens)
logger.info('org %s model %s, input_tokens %s, cached_tokens %s, output_tokens %s, cost %s', org_id, model, input_tokens, cached_tokens, output_tokens, cost)

params = {
'org_id': org_id,
'month': month,
'model': model,
'input_tokens': input_tokens,
'cached_tokens': cached_tokens,
'output_tokens': output_tokens,
'cost': input_cost + output_cost,
'cost': cost,
'created_at': datetime.now(),
'updated_at': datetime.now()
}
team_data.append(params)

owner_data = []
owner_sql = '''
INSERT INTO `stats_ai_by_owner`(`owner`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`)
VALUES (:owner, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at)
INSERT INTO `stats_ai_by_owner`(`owner`, `month`, `model`, `input_tokens`, `cached_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`)
VALUES (:owner, :month, :model, :input_tokens, :cached_tokens, :output_tokens, :cost, :created_at, :updated_at)
'''
for owner_id, models_dict in owner_stats.items():
for model, usage in models_dict.items():
input_tokens = usage.get('input_tokens') or 0
cached_tokens = usage.get('cached_tokens') or 0
output_tokens = usage.get('output_tokens') or 0

input_tokens_price = AI_PRICES[model].get('input_tokens') or 0
output_tokens_price = AI_PRICES[model].get('output_tokens') or 0
input_cost = input_tokens_price * (input_tokens / 1000000)
output_cost = output_tokens_price * (output_tokens / 1000000)
logger.info('owner %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', owner_id, model, input_tokens, input_cost, output_tokens, output_cost)
cost = calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens)
logger.info('owner %s model %s, input_tokens %s, cached_tokens %s, output_tokens %s, cost %s', owner_id, model, input_tokens, cached_tokens, output_tokens, cost)

params = {
'owner': owner_id,
'month': month,
'model': model,
'input_tokens': input_tokens,
'cached_tokens': cached_tokens,
'output_tokens': output_tokens,
'cost': input_cost + output_cost,
'cost': cost,
'created_at': datetime.now(),
'updated_at': datetime.now()
}
owner_data.append(params)

dtable_data = []
dtable_sql = '''
INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `group_id`, `org_id`, `scenario`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`)
VALUES (:dtable_uuid, :date, :model, :owner, :group_id, :org_id, :scenario, :input_tokens, :output_tokens, :cost, :created_at, :updated_at)
INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `group_id`, `org_id`, `scenario`, `input_tokens`, `cached_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`)
VALUES (:dtable_uuid, :date, :model, :owner, :group_id, :org_id, :scenario, :input_tokens, :cached_tokens, :output_tokens, :cost, :created_at, :updated_at)
ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`),
`cached_tokens`=`cached_tokens`+VALUES(`cached_tokens`),
`output_tokens`=`output_tokens`+VALUES(`output_tokens`),
`cost`=`cost`+VALUES(`cost`),
`updated_at`=VALUES(`updated_at`)
Expand All @@ -281,13 +299,11 @@ def stats_worker(self):

for (model, scenario), usage in models_dict.items():
input_tokens = usage.get('input_tokens') or 0
cached_tokens = usage.get('cached_tokens') or 0
output_tokens = usage.get('output_tokens') or 0

input_tokens_price = AI_PRICES[model].get('input_tokens') or 0
output_tokens_price = AI_PRICES[model].get('output_tokens') or 0
input_cost = input_tokens_price * (input_tokens / 1000000)
output_cost = output_tokens_price * (output_tokens / 1000000)
logger.info('dtable %s model %s scenario %s, input_tokens %s cost %s, output_tokens %s cost %s', dtable_uuid, model, scenario, input_tokens, input_cost, output_tokens, output_cost)
cost = calculate_usage_cost(model, input_tokens, cached_tokens, output_tokens)
logger.info('dtable %s model %s scenario %s, input_tokens %s, cached_tokens %s, output_tokens %s, cost %s', dtable_uuid, model, scenario, input_tokens, cached_tokens, output_tokens, cost)

params = {
'dtable_uuid': dtable_uuid,
Expand All @@ -298,8 +314,9 @@ def stats_worker(self):
'org_id': org_id,
'scenario': scenario,
'input_tokens': input_tokens,
'cached_tokens': cached_tokens,
'output_tokens': output_tokens,
'cost': input_cost + output_cost,
'cost': cost,
'created_at': datetime.now(),
'updated_at': datetime.now()
}
Expand Down
93 changes: 93 additions & 0 deletions tests/test_cached_tokens.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
"""Exercise statistics without initializing deployment config, Redis or MySQL."""

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Warning] 新计价测试未接入 CI

影响:当前工作流仅调用 test.sh 的 SQL 测试和 dtable_events.webhook.tests,不会发现此目录的四项新测试;本次成功的 CI 日志也只有原有 6 项和 16 项测试。因此缓存扣费、默认价格及三表写入回归仍可能在 CI 全绿时漏过。

建议修复:在工作流或 run_tests 中显式执行 python -m unittest discover -s tests -p test_cached_tokens.py -v,并让失败退出码传递给 Actions。

import ast
from collections import defaultdict
from copy import deepcopy
from datetime import datetime
import logging
from pathlib import Path
import unittest
from unittest.mock import MagicMock


SOURCE = Path(__file__).resolve().parents[1] / 'dtable_events'


def load_definitions(path, names, namespace):
tree = ast.parse(path.read_text())
tree.body = [node for node in tree.body if isinstance(node, (ast.ClassDef, ast.FunctionDef)) and node.name in names]
exec(compile(tree, str(path), 'exec'), namespace)
return namespace


class CachedTokensTest(unittest.TestCase):
def setUp(self):
self.prices = {'test-model': {'input_tokens': 2, 'cached_tokens': 0.5, 'output_tokens': 4}}
self.namespace = load_definitions(SOURCE / 'tasks/ai_stats_worker.py',
{'AIStatsWorker', 'calculate_usage_cost'}, {
'AI_PRICES': self.prices,
'defaultdict': defaultdict,
'deepcopy': deepcopy,
'datetime': datetime,
'logger': logging.getLogger(__name__),
'text': lambda sql: sql,
'uuid_str_to_32_chars': lambda uuid: uuid.replace('-', ''),
})
worker_class = self.namespace['AIStatsWorker']
self.worker = worker_class.__new__(worker_class)
self.worker.reset_stats()
self.worker._get_dtable_owner_info = MagicMock(return_value={'owner': 'owner', 'org_id': 1})

def save(self, cached_tokens=None, canonical=False):
usage = {'input_tokens': 1000, 'output_tokens': 100} if canonical else {'prompt_tokens': 1000, 'completion_tokens': 100}
if cached_tokens is not None:
usage['cached_tokens'] = cached_tokens
self.worker.save_to_memory({
'model': 'test-model', 'dtable_uuid': 'base', 'usage': usage,
'scenario': 'app_page_generation',
}, MagicMock())

def test_normalize_and_aggregate(self):
for value, expected in [(None, 0), (0, 0), (400, 400), (1000, 1000), (-10, 0), (2000, 1000), ('bad', 0)]:
with self.subTest(cached_tokens=value):
self.worker.reset_stats()
self.save(value)
self.save(value, canonical=True)
expected_usage = {'input_tokens': 2000, 'cached_tokens': expected * 2, 'output_tokens': 200}
self.assertEqual(self.worker.org_stats[1]['test-model'], expected_usage)
self.assertEqual(self.worker.owner_stats['owner']['test-model'], expected_usage)
self.assertEqual(self.worker.dtable_stats['base'][('test-model', 'app_page_generation')], expected_usage)

def test_cost_and_default_price(self):
calculate = self.namespace['calculate_usage_cost']
for cached, expected in [(0, 0.0024), (400, 0.0018), (1000, 0.0009)]:
self.assertAlmostEqual(calculate('test-model', 1000, cached, 100), expected)
self.prices['test-model']['cached_tokens'] = 0
self.assertAlmostEqual(calculate('test-model', 1000, 400, 100), 0.0016)
del self.prices['test-model']['cached_tokens']
self.assertAlmostEqual(calculate('test-model', 1000, 400, 100), 0.0024)

def test_flush_all_three_statistics_tables(self):
self.save(400)
self.worker.stats_lock = MagicMock()
self.worker.query_dtable_owners = MagicMock(return_value={'base': {'owner': 'owner', 'org_id': 1}})
session = MagicMock()
self.worker._db_session_class = MagicMock(return_value=session)
self.worker.stats_worker()
self.assertEqual(session.execute.call_count, 3)
for call in session.execute.call_args_list:
sql, rows = call.args
self.assertIn('`cached_tokens`', sql)
self.assertEqual(rows[0]['cached_tokens'], 400)
self.assertAlmostEqual(rows[0]['cost'], 0.0018)
session.commit.assert_called_once()

def test_price_configuration(self):
parse = load_definitions(SOURCE / 'app/config.py', {'get_llm_prices'}, {})['get_llm_prices']
for price, expected in [(2, 2), ({'input_tokens': 2}, 2), ({'input_tokens': 2, 'cached_tokens': 0}, 0),
({'input_tokens': 2, 'cached_tokens': 0.5}, 0.5)]:
with self.subTest(price=price):
self.assertEqual(parse([{'model': 'test-model', 'price': price}])['test-model']['cached_tokens'], expected)


if __name__ == '__main__':
unittest.main()
Loading