abap-llm/train/data/stats.json

{
 "corpus": "/Users/erhankeseli/projects/abap-llm/corpus/corpus.jsonl",
 "seed": 20261003,
 "max_len": 16384,
 "estimate_tokens": 2636112,
 "real_tokens": 2806501,
 "ratio_real_to_estimate": 1.065,
 "all": {
  "docs": 370,
  "tokens": 2806501,
  "min": 49,
  "p50": 5195,
  "p90": 16923,
  "p99": 21483,
  "max": 31552
 },
 "kept": {
  "docs": 325,
  "tokens": 1966365,
  "min": 49,
  "p50": 4027,
  "p90": 14998,
  "p99": 16217,
  "max": 16273
 },
 "train": {
  "docs": 309,
  "tokens": 1886475,
  "min": 49,
  "p50": 4403,
  "p90": 14998,
  "p99": 16217,
  "max": 16273
 },
 "valid": {
  "docs": 16,
  "tokens": 79890,
  "min": 130,
  "p50": 3392,
  "p90": 16027,
  "p99": 16129,
  "max": 16129
 },
 "removed": [
  {
   "objects": [
    "ZCL_DEMO_ABAP_OBJECTS (CLAS)"
   ],
   "tokens": 17042
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_STRUCTURES (CLAS)"
   ],
   "tokens": 16628
  },
  {
   "objects": [
    "32_Performance_Notes__P03 (DOC)"
   ],
   "tokens": 16742
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
   ],
   "tokens": 19117
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
   ],
   "tokens": 18883
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V755 (CLAS)"
   ],
   "tokens": 19801
  },
  {
   "objects": [
    "23_Date_and_Time (DOC)"
   ],
   "tokens": 20347
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES__V757 (CLAS)"
   ],
   "tokens": 17477
  },
  {
   "objects": [
    "07_String_Processing__P02 (DOC)"
   ],
   "tokens": 17916
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_CONSTRUCTOR_EXPR (CLAS)"
   ],
   "tokens": 17368
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V758 (CLAS)"
   ],
   "tokens": 31552
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_STRING_PROC__V757 (CLAS)"
   ],
   "tokens": 19499
  },
  {
   "objects": [
    "04_ABAP_Object_Orientation__P06 (DOC)"
   ],
   "tokens": 16694
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V758 (CLAS)"
   ],
   "tokens": 18368
  },
  {
   "objects": [
    "06_Dynamic_Programming__P07 (DOC)"
   ],
   "tokens": 23994
  },
  {
   "objects": [
    "28_Regular_Expressions (DOC)"
   ],
   "tokens": 17900
  },
  {
   "objects": [
    "06_Dynamic_Programming__P06 (DOC)"
   ],
   "tokens": 16923
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_NUMERIC_OP__V816 (CLAS)"
   ],
   "tokens": 19769
  },
  {
   "objects": [
    "05_Constructor_Expressions__P01 (DOC)"
   ],
   "tokens": 17349
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V755 (CLAS)"
   ],
   "tokens": 17237
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V756 (CLAS)"
   ],
   "tokens": 18069
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
   ],
   "tokens": 17923
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
   ],
   "tokens": 16924
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
   ],
   "tokens": 16770
  },
  {
   "objects": [
    "14_ABAP_Unit_Tests__P03 (DOC)"
   ],
   "tokens": 18549
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_STRING_PROC__V758 (CLAS)"
   ],
   "tokens": 22096
  },
  {
   "objects": [
    "22_Released_ABAP_Classes__P01 (DOC)"
   ],
   "tokens": 19022
  },
  {
   "objects": [
    "21_XML_JSON__P01 (DOC)"
   ],
   "tokens": 16625
  },
  {
   "objects": [
    "29_Numeric_Operations__P01 (DOC)"
   ],
   "tokens": 17805
  },
  {
   "objects": [
    "03_ABAP_SQL__P01 (DOC)"
   ],
   "tokens": 17305
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DTYPE_DOBJ__V757 (CLAS)"
   ],
   "tokens": 18308
  },
  {
   "objects": [
    "14_ABAP_Unit_Tests__P04 (DOC)"
   ],
   "tokens": 17633
  },
  {
   "objects": [
    "24_Builtin_Functions (DOC)"
   ],
   "tokens": 16851
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_NUMERIC_OP (CLAS)"
   ],
   "tokens": 19740
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES__V758 (CLAS)"
   ],
   "tokens": 17355
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_BUILTIN_FUNC (CLAS)"
   ],
   "tokens": 20446
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_STRING_PROC (CLAS)"
   ],
   "tokens": 20311
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_UNIT_TEST__V758 (CLAS)"
   ],
   "tokens": 16394
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V757 (CLAS)"
   ],
   "tokens": 21483
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DATE_TIME (CLAS)"
   ],
   "tokens": 18634
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_XML_JSON (CLAS)"
   ],
   "tokens": 16825
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_SQL (CLAS)"
   ],
   "tokens": 18283
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_INTERNAL_TABLES__V755 (CLAS)"
   ],
   "tokens": 17309
  },
  {
   "objects": [
    "ZCL_DEMO_ABAP_DYNAMIC_PROG__V756 (CLAS)"
   ],
   "tokens": 20979
  },
  {
   "objects": [
    "16_Data_Types_and_Objects__P02 (DOC)"
   ],
   "tokens": 17891
  }
 ],
 "by_type": {
  "CLAS": {
   "docs": 147,
   "tokens": 827969
  },
  "mixed(4)": {
   "docs": 6,
   "tokens": 28405
  },
  "DOC": {
   "docs": 106,
   "tokens": 1012152
  },
  "INTF": {
   "docs": 7,
   "tokens": 1792
  },
  "BDEF": {
   "docs": 6,
   "tokens": 1564
  },
  "DDLS": {
   "docs": 27,
   "tokens": 20614
  },
  "mixed(3)": {
   "docs": 2,
   "tokens": 6667
  },
  "PROG": {
   "docs": 13,
   "tokens": 55365
  },
  "DDLX": {
   "docs": 6,
   "tokens": 5003
  },
  "CLAS+DDLS": {
   "docs": 1,
   "tokens": 4563
  },
  "DCLS": {
   "docs": 2,
   "tokens": 245
  },
  "FUGR": {
   "docs": 2,
   "tokens": 2026
  }
 },
 "source": [
  "abap-cheat-sheets"
 ],
 "iterations_2_epochs": 618
}