{
 "corpus": "/Users/erhankeseli/projects/abap-llm/corpus/corpus.jsonl",
 "seed": 20261003,
 "max_len": 16384,
 "estimate_tokens": 2636112,
 "records": 370,
 "tokens_before_dedup": 2806501,
 "tokens_after_dedup": 2699110,
 "records_after_dedup": 351,
 "tokens_after_split": 2701996,
 "objects_with_versions": 39,
 "families": 269,
 "versions_kept": 319,
 "versions_removed": 16,
 "strict_rule_would_remove_more": 8,
 "removed_versions": [
  {
   "object": "ZBP_DEMO_ABAP_RAP_RO_U (CLAS)",
   "version": "v758",
   "newest": "main",
   "diff": 0.015,
   "tokens": 10696
  },
  {
   "object": "ZDEMO_ABAP_RAP_RO_M (BDEF)",
   "version": "v756",
   "newest": "main",
   "diff": 0.037,
   "tokens": 259
  },
  {
   "object": "ZDEMO_ABAP_RAP_RO_U (BDEF)",
   "version": "v756",
   "newest": "main",
   "diff": 0.043,
   "tokens": 236
  },
  {
   "object": "ZDEMO_ABAP_CDS_VE_ASSOC_E (DDLS)",
   "version": "v757",
   "newest": "main",
   "diff": 0.026,
   "tokens": 542
  },
  {
   "object": "ZDEMO_ABAP_CDS_VE_JOINS (DDLS)",
   "version": "v757",
   "newest": "main",
   "diff": 0.009,
   "tokens": 1827
  },
  {
   "object": "ZDEMO_ABAP_CDS_VE_AGG_EXP (DDLS)",
   "version": "v757",
   "newest": "main",
   "diff": 0.019,
   "tokens": 804
  },
  {
   "object": "ZDEMO_ABAP_CDS_VE_SEL (DDLS)",
   "version": "v757",
   "newest": "main",
   "diff": 0.029,
   "tokens": 3273
  },
  {
   "object": "ZDEMO_ABAP_CDS_VE_ASSOC (DDLS)",
   "version": "v757",
   "newest": "main",
   "diff": 0.008,
   "tokens": 2451
  },
  {
   "object": "ZDEMO_ABAP_FLI_VE (DDLS)",
   "version": "v816",
   "newest": "main",
   "diff": 0.0,
   "tokens": 168
  },
  {
   "object": "ZCL_DEMO_ABAP_DTYPE_DOBJ (CLAS)",
   "version": "v816",
   "newest": "main",
   "diff": 0.009,
   "tokens": 27407
  },
  {
   "object": "ZCL_DEMO_ABAP_DISPLAY (CLAS)",
   "version": "v755",
   "newest": "v757",
   "diff": 0.02,
   "tokens": 1677
  },
  {
   "object": "ZCL_DEMO_ABAP_UNIT_TEST (CLAS)",
   "version": "v758",
   "newest": "main",
   "diff": 0.034,
   "tokens": 19799
  },
  {
   "object": "ZDEMO_ABAP_DYNPRO (PROG)",
   "version": "v757",
   "newest": "v816",
   "diff": 0.001,
   "tokens": 8026
  },
  {
   "object": "ZBP_DEMO_ABAP_RAP_DRAFT_M (CLAS)",
   "version": "v757",
   "newest": "v758",
   "diff": 0.005,
   "tokens": 2474
  },
  {
   "object": "ZCL_DEMO_ABAP_NUMERIC_OP (CLAS)",
   "version": "v816",
   "newest": "main",
   "diff": 0.003,
   "tokens": 20484
  },
  {
   "object": "ZDEMO_ABAP_ALV (PROG)",
   "version": "v757",
   "newest": "v816",
   "diff": 0.002,
   "tokens": 7268
  }
 ],
 "split_docs": 43,
 "hard_line_splits": 17,
 "pieces_over_limit": 0,
 "train": {
  "docs": 374,
  "tokens": 2567180,
  "min": 49,
  "p50": 4904,
  "p90": 15638,
  "p99": 16177,
  "max": 16252
 },
 "valid": {
  "docs": 20,
  "tokens": 134816,
  "min": 92,
  "p50": 7589,
  "p90": 15317,
  "p99": 16273,
  "max": 16273
 },
 "all": {
  "docs": 394,
  "tokens": 2701996,
  "min": 49,
  "p50": 4904,
  "p90": 15608,
  "p99": 16217,
  "max": 16273
 },
 "share_before_dedup": {
  "CLAS": [
   1378559,
   49.1
  ],
  "other": [
   126244,
   4.5
  ],
  "DOC": [
   1301698,
   46.4
  ]
 },
 "share_final": {
  "CLAS": [
   1297878,
   48.0
  ],
  "other": [
   101390,
   3.8
  ],
  "DOC": [
   1302728,
   48.2
  ]
 },
 "by_type": {
  "CLAS": {
   "docs": 194,
   "tokens": 1297878
  },
  "mixed(4)": {
   "docs": 6,
   "tokens": 28405
  },
  "DOC": {
   "docs": 138,
   "tokens": 1302728
  },
  "INTF": {
   "docs": 7,
   "tokens": 1792
  },
  "BDEF": {
   "docs": 4,
   "tokens": 1069
  },
  "DDLS": {
   "docs": 21,
   "tokens": 11549
  },
  "mixed(3)": {
   "docs": 2,
   "tokens": 6667
  },
  "PROG": {
   "docs": 11,
   "tokens": 40071
  },
  "DDLX": {
   "docs": 6,
   "tokens": 5003
  },
  "CLAS+DDLS": {
   "docs": 1,
   "tokens": 4563
  },
  "DCLS": {
   "docs": 2,
   "tokens": 245
  },
  "FUGR": {
   "docs": 2,
   "tokens": 2026
  }
 },
 "split_info": [
  {
   "object": [
    "ZCL_DEMO_ABAP_OBJECTS (CLAS)"
   ],
   "tokens": 17042,
   "pieces": 2,
   "piece_tokens": [
    14777,
    2342
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_STRUCTURES (CLAS)"
   ],
   "tokens": 16628,
   "pieces": 2,
   "piece_tokens": [
    14200,
    2503
   ]
  },
  {
   "object": [
    "32_Performance_Notes__P03 (DOC)"
   ],
   "tokens": 16742,
   "pieces": 2,
   "piece_tokens": [
    15846,
    961
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
   ],
   "tokens": 19117,
   "pieces": 2,
   "piece_tokens": [
    14425,
    4770
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
   ],
   "tokens": 18883,
   "pieces": 2,
   "piece_tokens": [
    14607,
    4324
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V755 (CLAS)"
   ],
   "tokens": 19801,
   "pieces": 2,
   "piece_tokens": [
    15826,
    4036
   ]
  },
  {
   "object": [
    "23_Date_and_Time (DOC)"
   ],
   "tokens": 20347,
   "pieces": 2,
   "piece_tokens": [
    10310,
    10098
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES__V757 (CLAS)"
   ],
   "tokens": 17477,
   "pieces": 2,
   "piece_tokens": [
    2125,
    15451
   ]
  },
  {
   "object": [
    "07_String_Processing__P02 (DOC)"
   ],
   "tokens": 17916,
   "pieces": 2,
   "piece_tokens": [
    11096,
    6879
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_CONSTRUCTOR_EXPR (CLAS)"
   ],
   "tokens": 17368,
   "pieces": 2,
   "piece_tokens": [
    14031,
    3415
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V758 (CLAS)"
   ],
   "tokens": 31552,
   "pieces": 2,
   "piece_tokens": [
    15833,
    15780
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_STRING_PROC__V757 (CLAS)"
   ],
   "tokens": 19499,
   "pieces": 2,
   "piece_tokens": [
    15831,
    3729
   ]
  },
  {
   "object": [
    "04_ABAP_Object_Orientation__P06 (DOC)"
   ],
   "tokens": 16694,
   "pieces": 2,
   "piece_tokens": [
    9045,
    7716
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V758 (CLAS)"
   ],
   "tokens": 18368,
   "pieces": 2,
   "piece_tokens": [
    15833,
    2600
   ]
  },
  {
   "object": [
    "06_Dynamic_Programming__P07 (DOC)"
   ],
   "tokens": 23994,
   "pieces": 2,
   "piece_tokens": [
    15836,
    8221
   ]
  },
  {
   "object": [
    "28_Regular_Expressions (DOC)"
   ],
   "tokens": 17900,
   "pieces": 2,
   "piece_tokens": [
    15031,
    2930
   ]
  },
  {
   "object": [
    "06_Dynamic_Programming__P06 (DOC)"
   ],
   "tokens": 16923,
   "pieces": 2,
   "piece_tokens": [
    15842,
    1144
   ]
  },
  {
   "object": [
    "05_Constructor_Expressions__P01 (DOC)"
   ],
   "tokens": 17349,
   "pieces": 2,
   "piece_tokens": [
    15147,
    2265
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V755 (CLAS)"
   ],
   "tokens": 17237,
   "pieces": 2,
   "piece_tokens": [
    15833,
    1469
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V756 (CLAS)"
   ],
   "tokens": 18069,
   "pieces": 2,
   "piece_tokens": [
    15842,
    2292
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
   ],
   "tokens": 17923,
   "pieces": 2,
   "piece_tokens": [
    15329,
    2672
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
   ],
   "tokens": 16924,
   "pieces": 2,
   "piece_tokens": [
    15749,
    1223
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
   ],
   "tokens": 16770,
   "pieces": 2,
   "piece_tokens": [
    14210,
    2608
   ]
  },
  {
   "object": [
    "14_ABAP_Unit_Tests__P03 (DOC)"
   ],
   "tokens": 18549,
   "pieces": 2,
   "piece_tokens": [
    15834,
    2782
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_STRING_PROC__V758 (CLAS)"
   ],
   "tokens": 22096,
   "pieces": 2,
   "piece_tokens": [
    15814,
    6342
   ]
  },
  {
   "object": [
    "22_Released_ABAP_Classes__P01 (DOC)"
   ],
   "tokens": 19022,
   "pieces": 2,
   "piece_tokens": [
    14661,
    4430
   ]
  },
  {
   "object": [
    "21_XML_JSON__P01 (DOC)"
   ],
   "tokens": 16625,
   "pieces": 2,
   "piece_tokens": [
    9855,
    6839
   ]
  },
  {
   "object": [
    "29_Numeric_Operations__P01 (DOC)"
   ],
   "tokens": 17805,
   "pieces": 2,
   "piece_tokens": [
    15849,
    2025
   ]
  },
  {
   "object": [
    "03_ABAP_SQL__P01 (DOC)"
   ],
   "tokens": 17305,
   "pieces": 2,
   "piece_tokens": [
    14168,
    3198
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V757 (CLAS)"
   ],
   "tokens": 18308,
   "pieces": 2,
   "piece_tokens": [
    15842,
    2531
   ]
  },
  {
   "object": [
    "14_ABAP_Unit_Tests__P04 (DOC)"
   ],
   "tokens": 17633,
   "pieces": 2,
   "piece_tokens": [
    15834,
    1866
   ]
  },
  {
   "object": [
    "24_Builtin_Functions (DOC)"
   ],
   "tokens": 16851,
   "pieces": 2,
   "piece_tokens": [
    12541,
    4365
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_NUMERIC_OP (CLAS)"
   ],
   "tokens": 19740,
   "pieces": 2,
   "piece_tokens": [
    15825,
    3964
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES__V758 (CLAS)"
   ],
   "tokens": 17355,
   "pieces": 2,
   "piece_tokens": [
    15832,
    1584
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_BUILTIN_FUNC (CLAS)"
   ],
   "tokens": 20446,
   "pieces": 2,
   "piece_tokens": [
    15454,
    5071
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_STRING_PROC (CLAS)"
   ],
   "tokens": 20311,
   "pieces": 2,
   "piece_tokens": [
    15590,
    4797
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V757 (CLAS)"
   ],
   "tokens": 21483,
   "pieces": 2,
   "piece_tokens": [
    15815,
    5730
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DATE_TIME (CLAS)"
   ],
   "tokens": 18634,
   "pieces": 2,
   "piece_tokens": [
    15577,
    3138
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_XML_JSON (CLAS)"
   ],
   "tokens": 16825,
   "pieces": 2,
   "piece_tokens": [
    15753,
    1154
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_SQL (CLAS)"
   ],
   "tokens": 18283,
   "pieces": 2,
   "piece_tokens": [
    12526,
    5831
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES__V755 (CLAS)"
   ],
   "tokens": 17309,
   "pieces": 2,
   "piece_tokens": [
    2124,
    15283
   ]
  },
  {
   "object": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V756 (CLAS)"
   ],
   "tokens": 20979,
   "pieces": 2,
   "piece_tokens": [
    15831,
    5210
   ]
  },
  {
   "object": [
    "16_Data_Types_and_Objects__P02 (DOC)"
   ],
   "tokens": 17891,
   "pieces": 2,
   "piece_tokens": [
    14421,
    3541
   ]
  }
 ],
 "iterations_2_epochs": 748
}