erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
35f0eb0e0cca3283e0b687076a3d6229d4ebd45e
abap-llm/train/data/stats.json

664 lines
9.5 KiB
JSON
Raw Blame History

{
"corpus": "/Users/erhankeseli/projects/abap-llm/corpus/corpus.jsonl",
"seed": 20261003,
"max_len": 16384,
"estimate_tokens": 2636112,
"records": 370,
"tokens_before_dedup": 2806501,
"tokens_after_dedup": 2658082,
"records_after_dedup": 353,
"tokens_after_split": 2660834,
"objects_with_versions": 39,
"families": 269,
"versions_kept": 323,
"versions_removed": 12,
"other_branch_comparisons": 0,
"removed_versions": [
{
"object": "ZCL_DEMO_ABAP_OBJECTS (CLAS)",
"version": "v755",
"newest": "v757",
"diff": 0.01,
"tokens": 14944
},
{
"object": "ZDEMO_ABAP_RAP_DRAFT_M (BDEF)",
"version": "v756",
"newest": "v757",
"diff": 0.033,
"tokens": 302
},
{
"object": "ZCL_DEMO_ABAP_DTYPE_DOBJ (CLAS)",
"version": "v756",
"newest": "v757",
"diff": 0.015,
"tokens": 21263
},
{
"object": "ZCL_DEMO_ABAP_DTYPE_DOBJ (CLAS)",
"version": "v755",
"newest": "v756",
"diff": 0.047,
"tokens": 20431
},
{
"object": "ZCL_DEMO_ABAP_DISPLAY (CLAS)",
"version": "v755",
"newest": "v757",
"diff": 0.02,
"tokens": 1677
},
{
"object": "ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)",
"version": "v755",
"newest": "v757",
"diff": 0.01,
"tokens": 17309
},
{
"object": "ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)",
"version": "v756",
"newest": "v757",
"diff": 0.021,
"tokens": 25883
},
{
"object": "ZCL_DEMO_ABAP_SQL (CLAS)",
"version": "v755",
"newest": "v757",
"diff": 0.035,
"tokens": 14949
},
{
"object": "ZCL_DEMO_ABAP_CONSTRUCTOR_EXPR (CLAS)",
"version": "v755",
"newest": "v757",
"diff": 0.048,
"tokens": 13893
},
{
"object": "ZDEMO_ABAP_DYNPRO (PROG)",
"version": "v757",
"newest": "v816",
"diff": 0.001,
"tokens": 8026
},
{
"object": "ZBP_DEMO_ABAP_RAP_DRAFT_M (CLAS)",
"version": "v757",
"newest": "v758",
"diff": 0.005,
"tokens": 2474
},
{
"object": "ZDEMO_ABAP_ALV (PROG)",
"version": "v757",
"newest": "v816",
"diff": 0.002,
"tokens": 7268
}
],
"split_docs": 41,
"hard_line_splits": 15,
"pieces_over_limit": 0,
"train": {
"docs": 374,
"tokens": 2525568,
"min": 49,
"p50": 4770,
"p90": 15590,
"p99": 16177,
"max": 16252
},
"valid": {
"docs": 20,
"tokens": 135266,
"min": 135,
"p50": 7589,
"p90": 15317,
"p99": 16273,
"max": 16273
},
"all": {
"docs": 394,
"tokens": 2660834,
"min": 49,
"p50": 4797,
"p90": 15577,
"p99": 16217,
"max": 16273
},
"share_before_dedup": {
"CLAS": [
1378559,
49.1
],
"other": [
126244,
4.5
],
"DOC": [
1301698,
46.4
]
},
"share_final": {
"CLAS": [
1247458,
46.9
],
"other": [
110648,
4.2
],
"DOC": [
1302728,
49.0
]
},
"by_type": {
"CLAS": {
"docs": 187,
"tokens": 1247458
},
"mixed(4)": {
"docs": 6,
"tokens": 28405
},
"DOC": {
"docs": 138,
"tokens": 1302728
},
"INTF": {
"docs": 7,
"tokens": 1792
},
"BDEF": {
"docs": 5,
"tokens": 1262
},
"DDLS": {
"docs": 27,
"tokens": 20614
},
"mixed(3)": {
"docs": 2,
"tokens": 6667
},
"PROG": {
"docs": 11,
"tokens": 40071
},
"DDLX": {
"docs": 6,
"tokens": 5003
},
"CLAS+DDLS": {
"docs": 1,
"tokens": 4563
},
"DCLS": {
"docs": 2,
"tokens": 245
},
"FUGR": {
"docs": 2,
"tokens": 2026
}
},
"split_info": [
{
"object": [
"ZCL_DEMO_ABAP_OBJECTS (CLAS)"
],
"tokens": 17042,
"pieces": 2,
"piece_tokens": [
14777,
2342
]
},
{
"object": [
"ZCL_DEMO_ABAP_STRUCTURES (CLAS)"
],
"tokens": 16628,
"pieces": 2,
"piece_tokens": [
14200,
2503
]
},
{
"object": [
"32_Performance_Notes__P03 (DOC)"
],
"tokens": 16742,
"pieces": 2,
"piece_tokens": [
15846,
961
]
},
{
"object": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
],
"tokens": 19117,
"pieces": 2,
"piece_tokens": [
14425,
4770
]
},
{
"object": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
],
"tokens": 18883,
"pieces": 2,
"piece_tokens": [
14607,
4324
]
},
{
"object": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG__V755 (CLAS)"
],
"tokens": 19801,
"pieces": 2,
"piece_tokens": [
15826,
4036
]
},
{
"object": [
"23_Date_and_Time (DOC)"
],
"tokens": 20347,
"pieces": 2,
"piece_tokens": [
10310,
10098
]
},
{
"object": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES__V757 (CLAS)"
],
"tokens": 17477,
"pieces": 2,
"piece_tokens": [
2125,
15451
]
},
{
"object": [
"07_String_Processing__P02 (DOC)"
],
"tokens": 17916,
"pieces": 2,
"piece_tokens": [
11096,
6879
]
},
{
"object": [
"ZCL_DEMO_ABAP_CONSTRUCTOR_EXPR (CLAS)"
],
"tokens": 17368,
"pieces": 2,
"piece_tokens": [
14031,
3415
]
},
{
"object": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG__V758 (CLAS)"
],
"tokens": 31552,
"pieces": 2,
"piece_tokens": [
15833,
15780
]
},
{
"object": [
"ZCL_DEMO_ABAP_STRING_PROC__V757 (CLAS)"
],
"tokens": 19499,
"pieces": 2,
"piece_tokens": [
15831,
3729
]
},
{
"object": [
"04_ABAP_Object_Orientation__P06 (DOC)"
],
"tokens": 16694,
"pieces": 2,
"piece_tokens": [
9045,
7716
]
},
{
"object": [
"ZCL_DEMO_ABAP_DTYPE_DOBJ__V758 (CLAS)"
],
"tokens": 18368,
"pieces": 2,
"piece_tokens": [
15833,
2600
]
},
{
"object": [
"06_Dynamic_Programming__P07 (DOC)"
],
"tokens": 23994,
"pieces": 2,
"piece_tokens": [
15836,
8221
]
},
{
"object": [
"28_Regular_Expressions (DOC)"
],
"tokens": 17900,
"pieces": 2,
"piece_tokens": [
15031,
2930
]
},
{
"object": [
"06_Dynamic_Programming__P06 (DOC)"
],
"tokens": 16923,
"pieces": 2,
"piece_tokens": [
15842,
1144
]
},
{
"object": [
"ZCL_DEMO_ABAP_NUMERIC_OP__V816 (CLAS)"
],
"tokens": 19769,
"pieces": 2,
"piece_tokens": [
15819,
4012
]
},
{
"object": [
"05_Constructor_Expressions__P01 (DOC)"
],
"tokens": 17349,
"pieces": 2,
"piece_tokens": [
15147,
2265
]
},
{
"object": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
],
"tokens": 17923,
"pieces": 2,
"piece_tokens": [
15329,
2672
]
},
{
"object": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
],
"tokens": 16924,
"pieces": 2,
"piece_tokens": [
15749,
1223
]
},
{
"object": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
],
"tokens": 16770,
"pieces": 2,
"piece_tokens": [
14210,
2608
]
},
{
"object": [
"14_ABAP_Unit_Tests__P03 (DOC)"
],
"tokens": 18549,
"pieces": 2,
"piece_tokens": [
15834,
2782
]
},
{
"object": [
"ZCL_DEMO_ABAP_STRING_PROC__V758 (CLAS)"
],
"tokens": 22096,
"pieces": 2,
"piece_tokens": [
15814,
6342
]
},
{
"object": [
"22_Released_ABAP_Classes__P01 (DOC)"
],
"tokens": 19022,
"pieces": 2,
"piece_tokens": [
14661,
4430
]
},
{
"object": [
"21_XML_JSON__P01 (DOC)"
],
"tokens": 16625,
"pieces": 2,
"piece_tokens": [
9855,
6839
]
},
{
"object": [
"29_Numeric_Operations__P01 (DOC)"
],
"tokens": 17805,
"pieces": 2,
"piece_tokens": [
15849,
2025
]
},
{
"object": [
"03_ABAP_SQL__P01 (DOC)"
],
"tokens": 17305,
"pieces": 2,
"piece_tokens": [
14168,
3198
]
},
{
"object": [
"ZCL_DEMO_ABAP_DTYPE_DOBJ__V757 (CLAS)"
],
"tokens": 18308,
"pieces": 2,
"piece_tokens": [
15842,
2531
]
},
{
"object": [
"14_ABAP_Unit_Tests__P04 (DOC)"
],
"tokens": 17633,
"pieces": 2,
"piece_tokens": [
15834,
1866
]
},
{
"object": [
"24_Builtin_Functions (DOC)"
],
"tokens": 16851,
"pieces": 2,
"piece_tokens": [
12541,
4365
]
},
{
"object": [
"ZCL_DEMO_ABAP_NUMERIC_OP (CLAS)"
],
"tokens": 19740,
"pieces": 2,
"piece_tokens": [
15825,
3964
]
},
{
"object": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES__V758 (CLAS)"
],
"tokens": 17355,
"pieces": 2,
"piece_tokens": [
15832,
1584
]
},
{
"object": [
"ZCL_DEMO_ABAP_BUILTIN_FUNC (CLAS)"
],
"tokens": 20446,
"pieces": 2,
"piece_tokens": [
15454,
5071
]
},
{
"object": [
"ZCL_DEMO_ABAP_STRING_PROC (CLAS)"
],
"tokens": 20311,
"pieces": 2,
"piece_tokens": [
15590,
4797
]
},
{
"object": [
"ZCL_DEMO_ABAP_UNIT_TEST__V758 (CLAS)"
],
"tokens": 16394,
"pieces": 2,
"piece_tokens": [
15301,
1187
]
},
{
"object": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG__V757 (CLAS)"
],
"tokens": 21483,
"pieces": 2,
"piece_tokens": [
15815,
5730
]
},
{
"object": [
"ZCL_DEMO_ABAP_DATE_TIME (CLAS)"
],
"tokens": 18634,
"pieces": 2,
"piece_tokens": [
15577,
3138
]
},
{
"object": [
"ZCL_DEMO_ABAP_XML_JSON (CLAS)"
],
"tokens": 16825,
"pieces": 2,
"piece_tokens": [
15753,
1154
]
},
{
"object": [
"ZCL_DEMO_ABAP_SQL (CLAS)"
],
"tokens": 18283,
"pieces": 2,
"piece_tokens": [
12526,
5831
]
},
{
"object": [
"16_Data_Types_and_Objects__P02 (DOC)"
],
"tokens": 17891,
"pieces": 2,
"piece_tokens": [
14421,
3541
]
}
],
"iterations_2_epochs": 748
}
Reference in New Issue View Git Blame Copy Permalink