Update to 1M parameter model

README.md

@@ -0,0 +1,68 @@
+---
+tags:
+- generated_from_trainer
+datasets:
+- wikitext
+metrics:
+- accuracy
+model-index:
+- name: output_tiny
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: wikitext wikitext-103-v1
+      type: wikitext
+      args: wikitext-103-v1
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.2132901596611274
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# output_tiny
+
+This model is a fine-tuned version of [gpt2_tiny_random](https://huggingface.co/gpt2_tiny_random) on the wikitext wikitext-103-v1 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 5.3359
+- Accuracy: 0.2133
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 32
+- eval_batch_size: 64
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- training_steps: 50000
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.25.1
+- Pytorch 1.13.1
+- Datasets 2.8.0
+- Tokenizers 0.13.2

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 14.01,
+    "eval_accuracy": 0.2132901596611274,
+    "eval_loss": 5.335912704467773,
+    "eval_runtime": 3.848,
+    "eval_samples": 240,
+    "eval_samples_per_second": 62.37,
+    "eval_steps_per_second": 1.039,
+    "perplexity": 207.6621965544685,
+    "train_loss": 5.854185776367188,
+    "train_runtime": 10672.3435,
+    "train_samples": 114149,
+    "train_samples_per_second": 149.92,
+    "train_steps_per_second": 4.685
+}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "sshleifer/tiny-gpt2",
+  "_name_or_path": "gpt2_tiny_random",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -11,11 +11,10 @@
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt2",
-  "n_ctx": 1024,
-  "n_embd": 2,
+  "n_embd": 32,
   "n_head": 2,
-  "n_inner": null,
-  "n_layer": 2,
+  "n_inner": 64,
+  "n_layer": 4,
   "n_positions": 1024,
   "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
@@ -26,12 +25,6 @@
   "summary_proj_to_labels": true,
   "summary_type": "cls_index",
   "summary_use_proj": true,
-  "task_specific_params": {
-    "text-generation": {
-      "do_sample": true,
-      "max_length": 50
-    }
-  },
   "torch_dtype": "float32",
   "transformers_version": "4.25.1",
   "use_cache": true,

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 14.01,
+    "eval_accuracy": 0.2132901596611274,
+    "eval_loss": 5.335912704467773,
+    "eval_runtime": 3.848,
+    "eval_samples": 240,
+    "eval_samples_per_second": 62.37,
+    "eval_steps_per_second": 1.039,
+    "perplexity": 207.6621965544685
+}

optimizer.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f12c73d2dbcf4afe88abe24e15068670c968a0845c5ee50607cfd8403ef88905
-size 837233

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:597bf4a84749b25ad5bc5d2636b35453b7589b5b4c3f659e4ccecbd8108312c6
-size 2517777
+oid sha256:d18c7f041c7b7486dafd929d5b9ab0c88a537348e81f922e9fa397adc89f0eca
+size 10914405

rng_state.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7df3ae8280aa859e20e462d2e92adcc38d7b859521fb27c1a8ea01768b5899af
-size 14575

scheduler.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:98d33ab1f4c30262c17c55f5a0f085179cc0b0238fa1bff3cb54e5ff678cf9f9
-size 627

special_tokens_map.json

@@ -1,5 +1,23 @@
 {
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -9,7 +9,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     }
   ],

tokenizer_config.json

@@ -19,9 +19,9 @@
   },
   "errors": "replace",
   "model_max_length": 1024,
-  "name_or_path": "sshleifer/tiny-gpt2",
+  "name_or_path": "gpt2_tiny_random",
   "pad_token": null,
-  "special_tokens_map_file": "/root/.cache/huggingface/hub/models--sshleifer--tiny-gpt2/snapshots/5f91d94bd9cd7190a9f3216ff93cd1dd95f2c7be/special_tokens_map.json",
+  "special_tokens_map_file": null,
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": {
     "__type": "AddedToken",

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 14.01,
+    "train_loss": 5.854185776367188,
+    "train_runtime": 10672.3435,
+    "train_samples": 114149,
+    "train_samples_per_second": 149.92,
+    "train_steps_per_second": 4.685
+}

trainer_state.json

@@ -10,607 +10,616 @@
     {
       "epoch": 0.14,
       "learning_rate": 9.900000000000001e-05,
-      "loss": 10.747,
+      "loss": 9.7624,
       "step": 500
     },
     {
       "epoch": 0.28,
       "learning_rate": 9.8e-05,
-      "loss": 10.5732,
+      "loss": 8.2156,
       "step": 1000
     },
     {
       "epoch": 0.42,
       "learning_rate": 9.7e-05,
-      "loss": 10.3685,
+      "loss": 7.5205,
       "step": 1500
     },
     {
       "epoch": 0.56,
       "learning_rate": 9.6e-05,
-      "loss": 10.1396,
+      "loss": 7.3362,
       "step": 2000
     },
     {
       "epoch": 0.7,
       "learning_rate": 9.5e-05,
-      "loss": 9.8958,
+      "loss": 7.1343,
       "step": 2500
     },
     {
       "epoch": 0.84,
       "learning_rate": 9.4e-05,
-      "loss": 9.6456,
+      "loss": 6.9313,
       "step": 3000
     },
     {
       "epoch": 0.98,
       "learning_rate": 9.300000000000001e-05,
-      "loss": 9.3974,
+      "loss": 6.7408,
       "step": 3500
     },
     {
       "epoch": 1.12,
       "learning_rate": 9.200000000000001e-05,
-      "loss": 9.155,
+      "loss": 6.5858,
       "step": 4000
     },
     {
       "epoch": 1.26,
       "learning_rate": 9.1e-05,
-      "loss": 8.9232,
+      "loss": 6.4674,
       "step": 4500
     },
     {
       "epoch": 1.4,
       "learning_rate": 9e-05,
-      "loss": 8.7067,
+      "loss": 6.3797,
       "step": 5000
     },
     {
       "epoch": 1.54,
       "learning_rate": 8.900000000000001e-05,
-      "loss": 8.5025,
+      "loss": 6.3089,
       "step": 5500
     },
     {
       "epoch": 1.68,
       "learning_rate": 8.800000000000001e-05,
-      "loss": 8.3157,
+      "loss": 6.2535,
       "step": 6000
     },
     {
       "epoch": 1.82,
       "learning_rate": 8.7e-05,
-      "loss": 8.1445,
+      "loss": 6.2054,
       "step": 6500
     },
     {
       "epoch": 1.96,
       "learning_rate": 8.6e-05,
-      "loss": 7.9859,
+      "loss": 6.1582,
       "step": 7000
     },
     {
       "epoch": 2.1,
       "learning_rate": 8.5e-05,
-      "loss": 7.8496,
+      "loss": 6.1192,
       "step": 7500
     },
     {
       "epoch": 2.24,
       "learning_rate": 8.4e-05,
-      "loss": 7.7338,
+      "loss": 6.0872,
       "step": 8000
     },
     {
       "epoch": 2.38,
       "learning_rate": 8.3e-05,
-      "loss": 7.6332,
+      "loss": 6.0494,
       "step": 8500
     },
     {
       "epoch": 2.52,
       "learning_rate": 8.2e-05,
-      "loss": 7.5624,
+      "loss": 6.0233,
       "step": 9000
     },
     {
       "epoch": 2.66,
       "learning_rate": 8.1e-05,
-      "loss": 7.5056,
+      "loss": 5.993,
       "step": 9500
     },
     {
       "epoch": 2.8,
       "learning_rate": 8e-05,
-      "loss": 7.4691,
+      "loss": 5.9731,
       "step": 10000
     },
     {
       "epoch": 2.94,
       "learning_rate": 7.900000000000001e-05,
-      "loss": 7.446,
+      "loss": 5.9495,
       "step": 10500
     },
     {
       "epoch": 3.08,
       "learning_rate": 7.800000000000001e-05,
-      "loss": 7.4321,
+      "loss": 5.9287,
       "step": 11000
     },
     {
       "epoch": 3.22,
       "learning_rate": 7.7e-05,
-      "loss": 7.423,
+      "loss": 5.9077,
       "step": 11500
     },
     {
       "epoch": 3.36,
       "learning_rate": 7.6e-05,
-      "loss": 7.417,
+      "loss": 5.8884,
       "step": 12000
     },
     {
       "epoch": 3.5,
       "learning_rate": 7.500000000000001e-05,
-      "loss": 7.4149,
+      "loss": 5.8712,
       "step": 12500
     },
     {
       "epoch": 3.64,
       "learning_rate": 7.4e-05,
-      "loss": 7.4151,
+      "loss": 5.8562,
       "step": 13000
     },
     {
       "epoch": 3.78,
       "learning_rate": 7.3e-05,
-      "loss": 7.4099,
+      "loss": 5.836,
       "step": 13500
     },
     {
       "epoch": 3.92,
       "learning_rate": 7.2e-05,
-      "loss": 7.4135,
+      "loss": 5.8282,
       "step": 14000
     },
     {
       "epoch": 4.06,
       "learning_rate": 7.1e-05,
-      "loss": 7.4121,
+      "loss": 5.8108,
       "step": 14500
     },
     {
       "epoch": 4.2,
       "learning_rate": 7e-05,
-      "loss": 7.4121,
+      "loss": 5.7997,
       "step": 15000
     },
     {
       "epoch": 4.34,
       "learning_rate": 6.9e-05,
-      "loss": 7.4121,
+      "loss": 5.7869,
       "step": 15500
     },
     {
       "epoch": 4.48,
       "learning_rate": 6.800000000000001e-05,
-      "loss": 7.4097,
+      "loss": 5.7755,
       "step": 16000
     },
     {
       "epoch": 4.62,
       "learning_rate": 6.7e-05,
-      "loss": 7.4111,
+      "loss": 5.7686,
       "step": 16500
     },
     {
       "epoch": 4.76,
       "learning_rate": 6.6e-05,
-      "loss": 7.4112,
+      "loss": 5.7561,
       "step": 17000
     },
     {
       "epoch": 4.9,
       "learning_rate": 6.500000000000001e-05,
-      "loss": 7.4113,
+      "loss": 5.7481,
       "step": 17500
     },
     {
       "epoch": 5.04,
       "learning_rate": 6.400000000000001e-05,
-      "loss": 7.4102,
+      "loss": 5.7397,
       "step": 18000
     },
     {
       "epoch": 5.18,
       "learning_rate": 6.3e-05,
-      "loss": 7.4101,
+      "loss": 5.7285,
       "step": 18500
     },
     {
       "epoch": 5.33,
       "learning_rate": 6.2e-05,
-      "loss": 7.4114,
+      "loss": 5.7194,
       "step": 19000
     },
     {
       "epoch": 5.47,
       "learning_rate": 6.1e-05,
-      "loss": 7.4093,
+      "loss": 5.7117,
       "step": 19500
     },
     {
       "epoch": 5.61,
       "learning_rate": 6e-05,
-      "loss": 7.412,
+      "loss": 5.7066,
       "step": 20000
     },
     {
       "epoch": 5.75,
       "learning_rate": 5.9e-05,
-      "loss": 7.4086,
+      "loss": 5.6968,
       "step": 20500
     },
     {
       "epoch": 5.89,
       "learning_rate": 5.8e-05,
-      "loss": 7.4081,
+      "loss": 5.6889,
       "step": 21000
     },
     {
       "epoch": 6.03,
       "learning_rate": 5.6999999999999996e-05,
-      "loss": 7.4085,
+      "loss": 5.681,
       "step": 21500
     },
     {
       "epoch": 6.17,
       "learning_rate": 5.6000000000000006e-05,
-      "loss": 7.4107,
+      "loss": 5.6788,
       "step": 22000
     },
     {
       "epoch": 6.31,
       "learning_rate": 5.500000000000001e-05,
-      "loss": 7.4079,
+      "loss": 5.6668,
       "step": 22500
     },
     {
       "epoch": 6.45,
       "learning_rate": 5.4000000000000005e-05,
-      "loss": 7.4104,
+      "loss": 5.6649,
       "step": 23000
     },
     {
       "epoch": 6.59,
       "learning_rate": 5.300000000000001e-05,
-      "loss": 7.4088,
+      "loss": 5.6574,
       "step": 23500
     },
     {
       "epoch": 6.73,
       "learning_rate": 5.2000000000000004e-05,
-      "loss": 7.4087,
+      "loss": 5.6499,
       "step": 24000
     },
     {
       "epoch": 6.87,
       "learning_rate": 5.1000000000000006e-05,
-      "loss": 7.4079,
+      "loss": 5.6465,
       "step": 24500
     },
     {
       "epoch": 7.01,
       "learning_rate": 5e-05,
-      "loss": 7.4094,
+      "loss": 5.6419,
       "step": 25000
     },
     {
       "epoch": 7.15,
       "learning_rate": 4.9e-05,
-      "loss": 7.4098,
+      "loss": 5.6358,
       "step": 25500
     },
     {
       "epoch": 7.29,
       "learning_rate": 4.8e-05,
-      "loss": 7.4102,
+      "loss": 5.6312,
       "step": 26000
     },
     {
       "epoch": 7.43,
       "learning_rate": 4.7e-05,
-      "loss": 7.4107,
+      "loss": 5.6293,
       "step": 26500
     },
     {
       "epoch": 7.57,
       "learning_rate": 4.600000000000001e-05,
-      "loss": 7.4099,
+      "loss": 5.6218,
       "step": 27000
     },
     {
       "epoch": 7.71,
       "learning_rate": 4.5e-05,
-      "loss": 7.4086,
+      "loss": 5.618,
       "step": 27500
     },
     {
       "epoch": 7.85,
       "learning_rate": 4.4000000000000006e-05,
-      "loss": 7.4061,
+      "loss": 5.613,
       "step": 28000
     },
     {
       "epoch": 7.99,
       "learning_rate": 4.3e-05,
-      "loss": 7.407,
+      "loss": 5.6101,
       "step": 28500
     },
     {
       "epoch": 8.13,
       "learning_rate": 4.2e-05,
-      "loss": 7.4098,
+      "loss": 5.6048,
       "step": 29000
     },
     {
       "epoch": 8.27,
       "learning_rate": 4.1e-05,
-      "loss": 7.4087,
+      "loss": 5.6,
       "step": 29500
     },
     {
       "epoch": 8.41,
       "learning_rate": 4e-05,
-      "loss": 7.4079,
+      "loss": 5.5976,
       "step": 30000
     },
     {
       "epoch": 8.55,
       "learning_rate": 3.9000000000000006e-05,
-      "loss": 7.4076,
+      "loss": 5.5936,
       "step": 30500
     },
     {
       "epoch": 8.69,
       "learning_rate": 3.8e-05,
-      "loss": 7.4066,
+      "loss": 5.5931,
       "step": 31000
     },
     {
       "epoch": 8.83,
       "learning_rate": 3.7e-05,
-      "loss": 7.4105,
+      "loss": 5.5898,
       "step": 31500
     },
     {
       "epoch": 8.97,
       "learning_rate": 3.6e-05,
-      "loss": 7.4094,
+      "loss": 5.5888,
       "step": 32000
     },
     {
       "epoch": 9.11,
       "learning_rate": 3.5e-05,
-      "loss": 7.4101,
+      "loss": 5.582,
       "step": 32500
     },
     {
       "epoch": 9.25,
       "learning_rate": 3.4000000000000007e-05,
-      "loss": 7.4094,
+      "loss": 5.5817,
       "step": 33000
     },
     {
       "epoch": 9.39,
       "learning_rate": 3.3e-05,
-      "loss": 7.4067,
+      "loss": 5.5726,
       "step": 33500
     },
     {
       "epoch": 9.53,
       "learning_rate": 3.2000000000000005e-05,
-      "loss": 7.4103,
+      "loss": 5.5778,
       "step": 34000
     },
     {
       "epoch": 9.67,
       "learning_rate": 3.1e-05,
-      "loss": 7.4074,
+      "loss": 5.5718,
       "step": 34500
     },
     {
       "epoch": 9.81,
       "learning_rate": 3e-05,
-      "loss": 7.4091,
+      "loss": 5.569,
       "step": 35000
     },
     {
       "epoch": 9.95,
       "learning_rate": 2.9e-05,
-      "loss": 7.409,
+      "loss": 5.5703,
       "step": 35500
     },
     {
       "epoch": 10.09,
       "learning_rate": 2.8000000000000003e-05,
-      "loss": 7.4096,
+      "loss": 5.5668,
       "step": 36000
     },
     {
       "epoch": 10.23,
       "learning_rate": 2.7000000000000002e-05,
-      "loss": 7.4091,
+      "loss": 5.561,
       "step": 36500
     },
     {
       "epoch": 10.37,
       "learning_rate": 2.6000000000000002e-05,
-      "loss": 7.4053,
+      "loss": 5.5554,
       "step": 37000
     },
     {
       "epoch": 10.51,
       "learning_rate": 2.5e-05,
-      "loss": 7.4117,
+      "loss": 5.5621,
       "step": 37500
     },
     {
       "epoch": 10.65,
       "learning_rate": 2.4e-05,
-      "loss": 7.4081,
+      "loss": 5.558,
       "step": 38000
     },
     {
       "epoch": 10.79,
       "learning_rate": 2.3000000000000003e-05,
-      "loss": 7.4066,
+      "loss": 5.5557,
       "step": 38500
     },
     {
       "epoch": 10.93,
       "learning_rate": 2.2000000000000003e-05,
-      "loss": 7.4096,
+      "loss": 5.5572,
       "step": 39000
     },
     {
       "epoch": 11.07,
       "learning_rate": 2.1e-05,
-      "loss": 7.4072,
+      "loss": 5.5519,
       "step": 39500
     },
     {
       "epoch": 11.21,
       "learning_rate": 2e-05,
-      "loss": 7.4095,
+      "loss": 5.5502,
       "step": 40000
     },
     {
       "epoch": 11.35,
       "learning_rate": 1.9e-05,
-      "loss": 7.4074,
+      "loss": 5.5436,
       "step": 40500
     },
     {
       "epoch": 11.49,
       "learning_rate": 1.8e-05,
-      "loss": 7.4101,
+      "loss": 5.5482,
       "step": 41000
     },
     {
       "epoch": 11.63,
       "learning_rate": 1.7000000000000003e-05,
-      "loss": 7.4106,
+      "loss": 5.5524,
       "step": 41500
     },
     {
       "epoch": 11.77,
       "learning_rate": 1.6000000000000003e-05,
-      "loss": 7.4078,
+      "loss": 5.5449,
       "step": 42000
     },
     {
       "epoch": 11.91,
       "learning_rate": 1.5e-05,
-      "loss": 7.4063,
+      "loss": 5.5443,
       "step": 42500
     },
     {
       "epoch": 12.05,
       "learning_rate": 1.4000000000000001e-05,
-      "loss": 7.4094,
+      "loss": 5.5466,
       "step": 43000
     },
     {
       "epoch": 12.19,
       "learning_rate": 1.3000000000000001e-05,
-      "loss": 7.4071,
+      "loss": 5.5388,
       "step": 43500
     },
     {
       "epoch": 12.33,
       "learning_rate": 1.2e-05,
-      "loss": 7.407,
+      "loss": 5.5397,
       "step": 44000
     },
     {
       "epoch": 12.47,
       "learning_rate": 1.1000000000000001e-05,
-      "loss": 7.4092,
+      "loss": 5.5385,
       "step": 44500
     },
     {
       "epoch": 12.61,
       "learning_rate": 1e-05,
-      "loss": 7.4088,
+      "loss": 5.5437,
       "step": 45000
     },
     {
       "epoch": 12.75,
       "learning_rate": 9e-06,
-      "loss": 7.4092,
+      "loss": 5.541,
       "step": 45500
     },
     {
       "epoch": 12.89,
       "learning_rate": 8.000000000000001e-06,
-      "loss": 7.4099,
+      "loss": 5.5395,
       "step": 46000
     },
     {
       "epoch": 13.03,
       "learning_rate": 7.000000000000001e-06,
-      "loss": 7.4066,
+      "loss": 5.5357,
       "step": 46500
     },
     {
       "epoch": 13.17,
       "learning_rate": 6e-06,
-      "loss": 7.4069,
+      "loss": 5.5351,
       "step": 47000
     },
     {
       "epoch": 13.31,
       "learning_rate": 5e-06,
-      "loss": 7.4079,
+      "loss": 5.5353,
       "step": 47500
     },
     {
       "epoch": 13.45,
       "learning_rate": 4.000000000000001e-06,
-      "loss": 7.4085,
+      "loss": 5.5371,
       "step": 48000
     },
     {
       "epoch": 13.59,
       "learning_rate": 3e-06,
-      "loss": 7.4073,
+      "loss": 5.5343,
       "step": 48500
     },
     {
       "epoch": 13.73,
       "learning_rate": 2.0000000000000003e-06,
-      "loss": 7.4084,
+      "loss": 5.5371,
       "step": 49000
     },
     {
       "epoch": 13.87,
       "learning_rate": 1.0000000000000002e-06,
-      "loss": 7.4107,
+      "loss": 5.538,
       "step": 49500
     },
     {
       "epoch": 14.01,
       "learning_rate": 0.0,
-      "loss": 7.4089,
+      "loss": 5.5386,
       "step": 50000
+    },
+    {
+      "epoch": 14.01,
+      "step": 50000,
+      "total_flos": 336513375928320.0,
+      "train_loss": 5.854185776367188,
+      "train_runtime": 10672.3435,
+      "train_samples_per_second": 149.92,
+      "train_steps_per_second": 4.685
     }
   ],
   "max_steps": 50000,
   "num_train_epochs": 15,
-  "total_flos": 1493867790336.0,
+  "total_flos": 336513375928320.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:744c8b48ec3c78c469c26dc0c4d3a56c19e9bc5f491ebbdeded3f8d8cdddc06f
-size 3387
+oid sha256:f2348bfd80e07cc254ebc5b4b6d1946c8f12c082e277a8b4058e4cb3577e0fae
+size 3451