erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stage 1 step 1: prepare.py, real corpus numbers (SAP-samples/abap-cheat-sheets)

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-03 21:58:04 +02:00
parent f0593933f9
commit e083c9ca13
6 changed files with 593 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

370
train/data/stats.json Normal file
View File

@@ -0,0 +1,370 @@
{
"corpus": "/Users/erhankeseli/projects/abap-llm/corpus/corpus.jsonl",
"seed": 20261003,
"max_len": 16384,
"estimate_tokens": 2636112,
"real_tokens": 2806501,
"ratio_real_to_estimate": 1.065,
"all": {
"docs": 370,
"tokens": 2806501,
"min": 49,
"p50": 5195,
"p90": 16923,
"p99": 21483,
"max": 31552
},
"kept": {
"docs": 325,
"tokens": 1966365,
"min": 49,
"p50": 4027,
"p90": 14998,
"p99": 16217,
"max": 16273
},
"train": {
"docs": 309,
"tokens": 1886475,
"min": 49,
"p50": 4403,
"p90": 14998,
"p99": 16217,
"max": 16273
},
"valid": {
"docs": 16,
"tokens": 79890,
"min": 130,
"p50": 3392,
"p90": 16027,
"p99": 16129,
"max": 16129
},
"removed": [
{
"objects": [
"ZCL_DEMO_ABAP_OBJECTS (CLAS)"
],
"tokens": 17042
},
{
"objects": [
"ZCL_DEMO_ABAP_STRUCTURES (CLAS)"
],
"tokens": 16628
},
{
"objects": [
"32_Performance_Notes__P03 (DOC)"
],
"tokens": 16742
},
{
"objects": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
],
"tokens": 19117
},
{
"objects": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES (CLAS)"
],
"tokens": 18883
},
{
"objects": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG__V755 (CLAS)"
],
"tokens": 19801
},
{
"objects": [
"23_Date_and_Time (DOC)"
],
"tokens": 20347
},
{
"objects": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES__V757 (CLAS)"
],
"tokens": 17477
},
{
"objects": [
"07_String_Processing__P02 (DOC)"
],
"tokens": 17916
},
{
"objects": [
"ZCL_DEMO_ABAP_CONSTRUCTOR_EXPR (CLAS)"
],
"tokens": 17368
},
{
"objects": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG__V758 (CLAS)"
],
"tokens": 31552
},
{
"objects": [
"ZCL_DEMO_ABAP_STRING_PROC__V757 (CLAS)"
],
"tokens": 19499
},
{
"objects": [
"04_ABAP_Object_Orientation__P06 (DOC)"
],
"tokens": 16694
},
{
"objects": [
"ZCL_DEMO_ABAP_DTYPE_DOBJ__V758 (CLAS)"
],
"tokens": 18368
},
{
"objects": [
"06_Dynamic_Programming__P07 (DOC)"
],
"tokens": 23994
},
{
"objects": [
"28_Regular_Expressions (DOC)"
],
"tokens": 17900
},
{
"objects": [
"06_Dynamic_Programming__P06 (DOC)"
],
"tokens": 16923
},
{
"objects": [
"ZCL_DEMO_ABAP_NUMERIC_OP__V816 (CLAS)"
],
"tokens": 19769
},
{
"objects": [
"05_Constructor_Expressions__P01 (DOC)"
],
"tokens": 17349
},
{
"objects": [
"ZCL_DEMO_ABAP_DTYPE_DOBJ__V755 (CLAS)"
],
"tokens": 17237
},
{
"objects": [
"ZCL_DEMO_ABAP_DTYPE_DOBJ__V756 (CLAS)"
],
"tokens": 18069
},
{
"objects": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
],
"tokens": 17923
},
{
"objects": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
],
"tokens": 16924
},
{
"objects": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG (CLAS)"
],
"tokens": 16770
},
{
"objects": [
"14_ABAP_Unit_Tests__P03 (DOC)"
],
"tokens": 18549
},
{
"objects": [
"ZCL_DEMO_ABAP_STRING_PROC__V758 (CLAS)"
],
"tokens": 22096
},
{
"objects": [
"22_Released_ABAP_Classes__P01 (DOC)"
],
"tokens": 19022
},
{
"objects": [
"21_XML_JSON__P01 (DOC)"
],
"tokens": 16625
},
{
"objects": [
"29_Numeric_Operations__P01 (DOC)"
],
"tokens": 17805
},
{
"objects": [
"03_ABAP_SQL__P01 (DOC)"
],
"tokens": 17305
},
{
"objects": [
"ZCL_DEMO_ABAP_DTYPE_DOBJ__V757 (CLAS)"
],
"tokens": 18308
},
{
"objects": [
"14_ABAP_Unit_Tests__P04 (DOC)"
],
"tokens": 17633
},
{
"objects": [
"24_Builtin_Functions (DOC)"
],
"tokens": 16851
},
{
"objects": [
"ZCL_DEMO_ABAP_NUMERIC_OP (CLAS)"
],
"tokens": 19740
},
{
"objects": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES__V758 (CLAS)"
],
"tokens": 17355
},
{
"objects": [
"ZCL_DEMO_ABAP_BUILTIN_FUNC (CLAS)"
],
"tokens": 20446
},
{
"objects": [
"ZCL_DEMO_ABAP_STRING_PROC (CLAS)"
],
"tokens": 20311
},
{
"objects": [
"ZCL_DEMO_ABAP_UNIT_TEST__V758 (CLAS)"
],
"tokens": 16394
},
{
"objects": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG__V757 (CLAS)"
],
"tokens": 21483
},
{
"objects": [
"ZCL_DEMO_ABAP_DATE_TIME (CLAS)"
],
"tokens": 18634
},
{
"objects": [
"ZCL_DEMO_ABAP_XML_JSON (CLAS)"
],
"tokens": 16825
},
{
"objects": [
"ZCL_DEMO_ABAP_SQL (CLAS)"
],
"tokens": 18283
},
{
"objects": [
"ZCL_DEMO_ABAP_INTERNAL_TABLES__V755 (CLAS)"
],
"tokens": 17309
},
{
"objects": [
"ZCL_DEMO_ABAP_DYNAMIC_PROG__V756 (CLAS)"
],
"tokens": 20979
},
{
"objects": [
"16_Data_Types_and_Objects__P02 (DOC)"
],
"tokens": 17891
}
],
"by_type": {
"CLAS": {
"docs": 147,
"tokens": 827969
},
"mixed(4)": {
"docs": 6,
"tokens": 28405
},
"DOC": {
"docs": 106,
"tokens": 1012152
},
"INTF": {
"docs": 7,
"tokens": 1792
},
"BDEF": {
"docs": 6,
"tokens": 1564
},
"DDLS": {
"docs": 27,
"tokens": 20614
},
"mixed(3)": {
"docs": 2,
"tokens": 6667
},
"PROG": {
"docs": 13,
"tokens": 55365
},
"DDLX": {
"docs": 6,
"tokens": 5003
},
"CLAS+DDLS": {
"docs": 1,
"tokens": 4563
},
"DCLS": {
"docs": 2,
"tokens": 245
},
"FUGR": {
"docs": 2,
"tokens": 2026
}
},
"source": [
"abap-cheat-sheets"
],
"iterations_2_epochs": 618
}