{ "schema_version": 1, "measured_at": "2026-10-17T07:07:22.805174+00:00", "environment": { "python": "3.14.8", "system": "macOS-27.0.1-arm64-arm-64bit-Mach-O", "packages": { "tiktoken": "0.14.0", "google-genai": "2.31.0", "sentencepiece": "0.2.2", "protobuf": "7.36.2" } }, "method": "Plain text only; Unicode code point length including spaces/newlines; no NFC normalization, role headers, system prompts or tool schemas. Ratios are token_count / characters. No model inference or API request.", "tokenizers": [ { "model": "gpt-4o-mini", "implementation": "tiktoken.encoding_for_model", "encoding": "o200k_base", "source_url": "https://github.com/openai/tiktoken/blob/main/tiktoken/model.py", "model_mapping_code_sha256": "600f26902d1cf6a1a5f54e37be988b3e0d911f1ff17ba7060bb361f9b5295521", "vocabulary_sha256": "6b1b104fbdf5ea90370ff838e90173c0c01774c47a5ef98c470fb7375d65f386", "vocabulary_hash_format": "sorted JSON array of [piece.hex(), rank], compact separators" }, { "model": "gemini-2.5-pro", "implementation": "google.genai.local_tokenizer.LocalTokenizer", "experimental": true, "encoding": "gemma3", "source_url": "https://github.com/googleapis/python-genai/blob/main/google/genai/_local_tokenizer_loader.py", "model_mapping_code_sha256": "eff7d8520c4656655cd42d3a7e981093fbd9cee8017db255c58df5112174df3c", "vocabulary_sha256": "1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c", "vocabulary_hash_format": "SentencePiece serialized_model_proto bytes" } ], "unmeasured": [ { "company": "Anthropic", "status": "not_measured", "reason": "The official public legacy tokenizer is not accurate for Claude 3 onward; the documented current token-counting endpoint requires an API key. No key or legacy estimate used.", "sources": [ "https://github.com/anthropics/anthropic-tokenizer-typescript", "https://platform.claude.com/docs/en/build-with-claude/token-counting" ] } ], "limitations": [ "No inference quality, latency, cost or billing totals measured.", "No extrapolation to other models, including newer Gemini models or Claude.", "Google local tokenizer is experimental; these are local SDK counts, not observed API billing." ], "unit_map": { "results[].samples[].characters": "字", "results[].samples[].token_count": "トークン" }, "results": [ { "model": "gpt-4o-mini", "samples": [ { "id": "greeting", "label": "短い挨拶", "text": "こんにちは。今日はよろしくお願いします。", "text_sha256": "7fe646417fd582f15962ec0a2c3cc31d050e9451584caf02d31c87948eacd364", "characters": 20, "token_ids": [ 95839, 788, 170411, 131888, 137015, 788 ], "token_count": 6 }, { "id": "explanation", "label": "説明文", "text": "AIの利用料金は、文章の文字数だけでは決まりません。モデルは入力された文章をトークンという小さな単位に分けて処理します。同じ日本語でも、言葉の選び方や記号の使い方によって、必要なトークン数は変わります。料金を比べるときは、入力と出力の単価に加えて、実際に送る文章を数えることが大切です。この計測では、挨拶、説明文、表を含む文章を固定して、公開された道具で数えます。改行や空白も文章の一部として扱い、結果と元の文章を一緒に保存します。外部の生成サービスには文章を送らず、手元で数えられた結果だけを掲載します。測れない会社の値を、別の会社の結果から推測して埋めることはしません。比較の条件を必ず揃えます。", "text_sha256": "61f5e106bc8147037a96c80ac8fcb02030b44b882f9acf5e8c14f7011f4901b4", "characters": 300, "token_ids": [ 17527, 3385, 68132, 137048, 5205, 1395, 26474, 3385, 79831, 7135, 121885, 43147, 78755, 137077, 63366, 788, 187451, 5205, 195421, 188599, 26474, 7277, 5662, 77764, 4025, 67985, 5820, 7500, 5784, 87236, 9838, 5280, 2957, 15707, 5711, 158556, 5584, 38720, 189245, 29884, 9048, 40909, 45996, 1395, 17765, 86064, 3385, 46629, 58490, 4983, 19160, 22926, 4785, 3385, 15159, 3826, 4983, 158777, 30677, 1395, 125586, 5784, 5662, 77764, 4025, 7135, 5205, 60722, 20473, 57684, 788, 137048, 7277, 11826, 55078, 6996, 5330, 11852, 5205, 1395, 195421, 5330, 6390, 11343, 3385, 87236, 35816, 5280, 9116, 170450, 1395, 38048, 71005, 5280, 9783, 6996, 26474, 7277, 7135, 133169, 53217, 6632, 1640, 34921, 15121, 197837, 37662, 134925, 43147, 1395, 6695, 101, 4347, 114, 1395, 101127, 4883, 1395, 9314, 7277, 47373, 44286, 26474, 7277, 166937, 22440, 1395, 161600, 188599, 7615, 22911, 4344, 7135, 18606, 14429, 788, 15346, 6550, 19160, 16207, 17760, 8446, 26474, 3385, 2432, 8572, 176315, 161953, 3826, 1395, 109426, 5330, 6753, 3385, 26474, 7277, 2432, 168147, 5280, 66421, 38720, 788, 10727, 8572, 3385, 70033, 124256, 68857, 26474, 7277, 9783, 8870, 36743, 1395, 6034, 6753, 4344, 7135, 18606, 8870, 9472, 5598, 109426, 121885, 7277, 137149, 38720, 788, 134925, 9472, 47592, 81407, 3385, 69555, 7277, 1395, 47106, 3385, 81407, 3385, 109426, 27500, 11800, 134925, 22440, 6252, 233, 17693, 6996, 53217, 5205, 3829, 63366, 788, 196048, 3385, 59062, 7277, 20805, 36743, 5441, 225, 18606, 14429, 788 ], "token_count": 225 }, { "id": "table", "label": "表を含む文", "text": "次の表から、必要な道具を選んでください。\n| 作業 | 道具 |\n|---|---|\n| 下書き | メモ帳 |\n| 計算 | 表計算 |\n理由を短く説明してください。", "text_sha256": "e7fc789420032eed93dfa2dc59fc80c29563833c672b54f09567bcc02f8fa8fd", "characters": 84, "token_ids": [ 11445, 3385, 9314, 27500, 1395, 125586, 5784, 7615, 22911, 7277, 46629, 6676, 4344, 44900, 3414, 91, 150642, 23784, 1022, 57226, 241, 22911, 15972, 91, 10356, 91, 10356, 14876, 91, 119311, 33528, 11852, 1022, 60677, 37938, 146612, 15972, 91, 87136, 230, 16727, 1022, 110242, 37662, 16727, 15972, 133602, 7277, 40730, 10459, 101127, 148746, 788 ], "token_count": 53 } ] }, { "model": "gemini-2.5-pro", "samples": [ { "id": "greeting", "label": "短い挨拶", "text": "こんにちは。今日はよろしくお願いします。", "text_sha256": "7fe646417fd582f15962ec0a2c3cc31d050e9451584caf02d31c87948eacd364", "characters": 20, "token_ids": [ 85141, 236924, 90082, 196460, 236924 ], "token_count": 5 }, { "id": "explanation", "label": "説明文", "text": "AIの利用料金は、文章の文字数だけでは決まりません。モデルは入力された文章をトークンという小さな単位に分けて処理します。同じ日本語でも、言葉の選び方や記号の使い方によって、必要なトークン数は変わります。料金を比べるときは、入力と出力の単価に加えて、実際に送る文章を数えることが大切です。この計測では、挨拶、説明文、表を含む文章を固定して、公開された道具で数えます。改行や空白も文章の一部として扱い、結果と元の文章を一緒に保存します。外部の生成サービスには文章を送らず、手元で数えられた結果だけを掲載します。測れない会社の値を、別の会社の結果から推測して埋めることはしません。比較の条件を必ず揃えます。", "text_sha256": "61f5e106bc8147037a96c80ac8fcb02030b44b882f9acf5e8c14f7011f4901b4", "characters": 300, "token_ids": [ 12553, 236945, 16465, 88585, 237048, 236951, 30886, 236945, 31818, 237311, 14606, 9450, 238903, 30263, 13772, 236924, 35128, 237048, 66245, 17011, 30886, 237051, 186745, 237023, 11073, 117256, 101851, 237000, 226028, 49945, 11914, 236924, 32437, 94951, 13389, 236951, 69798, 236945, 131704, 237256, 237512, 238218, 238013, 185458, 30405, 236951, 80395, 186745, 237023, 110693, 55930, 13764, 236924, 88585, 237051, 125269, 202371, 237048, 236951, 66245, 237032, 106789, 236945, 239155, 238927, 237000, 144307, 236951, 103586, 237859, 237033, 30886, 237051, 237311, 15982, 14096, 87934, 3652, 236924, 8978, 213559, 9450, 236951, 150439, 236951, 34084, 237364, 236951, 237467, 85678, 30886, 237051, 48768, 3335, 236951, 60710, 17011, 102586, 237007, 237311, 66268, 236924, 238035, 237236, 237512, 177698, 237156, 30886, 114628, 12547, 110682, 236951, 27654, 237032, 110053, 30886, 237051, 77992, 30826, 11914, 236924, 49753, 236945, 25352, 39149, 10215, 30886, 197400, 57975, 236951, 237463, 237798, 237007, 237311, 237495, 38152, 27654, 14606, 237051, 81817, 11914, 236924, 239525, 49262, 185314, 142352, 236951, 95466, 29241, 147079, 5077, 238206, 239525, 3335, 241003, 26085, 41277, 126690, 236924, 32223, 236945, 23991, 237051, 70381, 241844, 66268, 236924 ], "token_count": 167 }, { "id": "table", "label": "表を含む文", "text": "次の表から、必要な道具を選んでください。\n| 作業 | 道具 |\n|---|---|\n| 下書き | メモ帳 |\n| 計算 | 表計算 |\n理由を短く説明してください。", "text_sha256": "e7fc789420032eed93dfa2dc59fc80c29563833c672b54f09567bcc02f8fa8fd", "characters": 84, "token_ids": [ 60070, 237467, 5077, 236951, 80395, 102586, 79256, 21455, 15142, 236924, 107, 236909, 173933, 1109, 236743, 102586, 1109, 107, 236909, 171168, 171168, 107, 236909, 32210, 60147, 1109, 31705, 237966, 239828, 1109, 107, 236909, 154360, 1109, 67121, 44068, 1109, 107, 42867, 237051, 238906, 237234, 34084, 49391, 236924 ], "token_count": 45 } ] } ] }