axolotl version: 0.4.1

base_model: /workspace/medius-erebus
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer


hubmodelid: magnum-erebus-14b-v1
hubstrategy: "allcheckpoints"
pushdatasetto_hub:
hfuseauth_token: true

plugins:
  - axolotl.integrations.liger.LigerPlugin
liger_rope: true
ligerrmsnorm: true
liger_swiglu: true
ligerfusedlinearcrossentropy: true

loadin8bit: false
loadin4bit: false
strict: false

datasets:
  - path: anthracite-core/c2logs32kllama3qwen2_v1.2
    type: sharegpt
  - path: anthracite-org/kalo-opus-instruct-22k-no-refusal
    type: sharegpt
  - path: lodrick-the-lafted/kalo-opus-instruct-3k-filtered
    type: sharegpt
  - path: anthracite-org/nopmclaudewriting_fixed
    type: sharegpt
  - path: anthracite-org/kaloopusmisc_240827
    type: sharegpt
  - path: anthracite-org/kalomiscpart2
    type: sharegpt
chat_template: chatml
shufflemergeddatasets: true
defaultsystemmessage: "You are an assistant that responds to the user."
datasetpreparedpath: /workspace/data/magnum-14b-data
valsetsize: 0.0
output_dir: /workspace/data/magnum-erebus-14b-fft

sequence_len: 32768
sample_packing: true
padtosequence_len: true

adapter:
loramodeldir:
lora_r:
lora_alpha:
lora_dropout:
loratargetlinear:
lorafaninfanout:

wandb_project: 14b-magnum-fft
wandb_entity:
wandb_watch:
wandb_name: v4-r2-erebus-attempt-1
wandblogmodel:

gradientaccumulationsteps: 1
microbatchsize: 2
num_epochs: 2
optimizer: adamwbnb8bit
lr_scheduler: cosine
learning_rate: 0.000008

trainoninputs: false
groupbylength: false
bf16: auto
fp16:
tf32: false

gradient_checkpointing: unsloth
earlystoppingpatience:
resumefromcheckpoint:
local_rank:
logging_steps: 1
xformers_attention:
flash_attention: true

warmup_steps: 40
evalsperepoch:
evaltablesize:
evalmaxnew_tokens:
savesperepoch: 2
debug:
deepspeed: deepspeedconfigs/zero3bf16.json
weight_decay: 0.1
fsdp:
fsdp_config:
special_tokens: