File size: 2,925 Bytes
5e76121
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
{
  "backend": "tokenizers",
  "begin_of_text": "<|begin_of_text|>",
  "clean_up_tokenization_spaces": false,
  "content_image": "<|content_image|>",
  "content_invoke_tool_json": "<|content_invoke_tool_json|>",
  "content_invoke_tool_text": "<|content_invoke_tool_text|>",
  "content_model_end_sampling": "<|content_model_end_sampling|>",
  "content_text": "<|content_text|>",
  "content_thinking": "<|content_thinking|>",
  "content_tool_error": "<|content_tool_error|>",
  "content_xml": "<|content_xml|>",
  "end_message": "<|end_message|>",
  "endoftext": "<|endoftext|>",
  "eos_token": "<|content_model_end_sampling|>",
  "fix_mistral_regex": false,
  "is_local": false,
  "local_files_only": false,
  "message_model": "<|message_model|>",
  "message_system": "<|message_system|>",
  "message_tool": "<|message_tool|>",
  "message_user": "<|message_user|>",
  "model_max_length": 1048576,
  "model_specific_special_tokens": {
    "begin_of_text": "<|begin_of_text|>",
    "content_image": "<|content_image|>",
    "content_invoke_tool_json": "<|content_invoke_tool_json|>",
    "content_invoke_tool_text": "<|content_invoke_tool_text|>",
    "content_model_end_sampling": "<|content_model_end_sampling|>",
    "content_text": "<|content_text|>",
    "content_thinking": "<|content_thinking|>",
    "content_tool_error": "<|content_tool_error|>",
    "content_xml": "<|content_xml|>",
    "end_message": "<|end_message|>",
    "endoftext": "<|endoftext|>",
    "message_model": "<|message_model|>",
    "message_system": "<|message_system|>",
    "message_tool": "<|message_tool|>",
    "message_user": "<|message_user|>"
  },
  "pad_token": "<|endoftext|>",
  "processor_class": "Inkling2Processor",
  "response_template": {
    "defaults": {
      "role": "assistant"
    },
    "fields": {
      "content": {
        "close": "<|end_message|>",
        "content": "text",
        "content_args": {
          "strip": false
        },
        "join": "",
        "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_(?:text|tool_error|invoke_tool_text)\\|>",
        "repeats": true
      },
      "reasoning_content": {
        "close": "<|end_message|>",
        "content": "text",
        "content_args": {
          "strip": false
        },
        "join": "",
        "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_thinking\\|>",
        "repeats": true
      },
      "tool_calls": {
        "close": "<|end_message|>",
        "content": "json",
        "open_pattern": "(?:<\\|message_model\\|>[^<]*)?<\\|content_invoke_tool_json\\|>",
        "repeats": true,
        "transform": {
          "function": {
            "arguments": "{content.args}",
            "name": "{content.name}"
          },
          "type": "function"
        }
      }
    },
    "start_anchor_pattern": "<\\|end_message\\|>|<\\|content_model_end_sampling\\|>"
  },
  "tokenizer_class": "TokenizersBackend"
}