Rewrite the model card; link the dataset; add temperatures.json and test answers
Browse files- README.md +290 -28
- eval/score.txt +23 -0
- eval/test-answers.jsonl +0 -0
- temperatures.json +19 -0
README.md
CHANGED
|
@@ -1,8 +1,14 @@
|
|
| 1 |
---
|
| 2 |
license: other
|
| 3 |
-
license_name:
|
| 4 |
license_link: https://huggingface.co/C-Tianyu/NanoJev
|
| 5 |
base_model: C-Tianyu/NanoJev
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
library_name: pytorch
|
| 7 |
pipeline_tag: text-classification
|
| 8 |
tags:
|
|
@@ -10,52 +16,308 @@ tags:
|
|
| 10 |
- typed-decisions
|
| 11 |
- system-one
|
| 12 |
- code-review
|
|
|
|
| 13 |
- javascript
|
| 14 |
- typescript
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
---
|
| 16 |
|
| 17 |
# NanoJev Code Review Decisions
|
| 18 |
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
## Results
|
| 24 |
|
| 25 |
-
|
| 26 |
-
scored with the dataset's `score_decisions.py`. Latency is the median time for one snippet (13 questions) alone on
|
| 27 |
-
one H100 SXM.
|
| 28 |
|
| 29 |
-
| Model | Macro F1 @ 0.5 | Accuracy | Log loss | Brier | ECE |
|
| 30 |
-
|---|---:|---:|---:|---:|---:|---:|
|
| 31 |
-
|
|
| 32 |
-
|
|
| 33 |
-
|
|
| 34 |
-
|
|
|
|
|
|
|
|
| 35 |
|
| 36 |
-
|
| 37 |
|
| 38 |
-
|
| 39 |
|
| 40 |
-
|
| 41 |
-
Load it with NanoJev's `DecisionPredictor` from the
|
| 42 |
-
[NanoJev release](https://huggingface.co/C-Tianyu/NanoJev) (revision `unified-games-v1`), or with
|
| 43 |
-
`training/nanojev.py predict --ckpt <folder>`, which also builds the 13 questions.
|
| 44 |
|
| 45 |
-
|
| 46 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
|
| 48 |
## Training
|
| 49 |
|
| 50 |
-
- Data: train split of
|
| 51 |
-
-
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
-
|
| 55 |
|
| 56 |
-
|
| 57 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
|
| 59 |
## License
|
| 60 |
|
| 61 |
-
Not
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
license: other
|
| 3 |
+
license_name: unspecified
|
| 4 |
license_link: https://huggingface.co/C-Tianyu/NanoJev
|
| 5 |
base_model: C-Tianyu/NanoJev
|
| 6 |
+
base_model_relation: finetune
|
| 7 |
+
datasets:
|
| 8 |
+
- yannelli/js-ts-code-review-decisions
|
| 9 |
+
language:
|
| 10 |
+
- en
|
| 11 |
+
- code
|
| 12 |
library_name: pytorch
|
| 13 |
pipeline_tag: text-classification
|
| 14 |
tags:
|
|
|
|
| 16 |
- typed-decisions
|
| 17 |
- system-one
|
| 18 |
- code-review
|
| 19 |
+
- code-quality
|
| 20 |
- javascript
|
| 21 |
- typescript
|
| 22 |
+
- nanojev
|
| 23 |
+
- qwen3
|
| 24 |
+
model-index:
|
| 25 |
+
- name: nanojev-code-review-decisions
|
| 26 |
+
results:
|
| 27 |
+
- task:
|
| 28 |
+
type: text-classification
|
| 29 |
+
name: Code review decisions
|
| 30 |
+
dataset:
|
| 31 |
+
name: JS/TS Code Review Decisions
|
| 32 |
+
type: yannelli/js-ts-code-review-decisions
|
| 33 |
+
config: decisions
|
| 34 |
+
split: test
|
| 35 |
+
metrics:
|
| 36 |
+
- type: f1
|
| 37 |
+
name: Macro F1 @ 0.5
|
| 38 |
+
value: 0.452
|
| 39 |
+
- type: accuracy
|
| 40 |
+
name: Accuracy
|
| 41 |
+
value: 0.831
|
| 42 |
+
- type: log_loss
|
| 43 |
+
name: Log loss
|
| 44 |
+
value: 0.459
|
| 45 |
+
- type: brier_score
|
| 46 |
+
name: Brier score
|
| 47 |
+
value: 0.173
|
| 48 |
+
- type: ece
|
| 49 |
+
name: Expected calibration error
|
| 50 |
+
value: 0.038
|
| 51 |
+
source:
|
| 52 |
+
name: score_decisions.py
|
| 53 |
+
url: https://huggingface.co/datasets/yannelli/js-ts-code-review-decisions
|
| 54 |
---
|
| 55 |
|
| 56 |
# NanoJev Code Review Decisions
|
| 57 |
|
| 58 |
+
A decision model that reviews JavaScript and TypeScript code. Give it a snippet and the task the snippet implements, and it answers 13 typed questions with probabilities: which of 10 code review flaws the code has, whether it has any flaw, which flaw matters most, and how mergeable it is. Every answer option is encoded as its own path with the snippet and question (27 paths per snippet), and the choice head scores the paths.
|
| 59 |
+
|
| 60 |
+
It is [C-Tianyu/NanoJev](https://huggingface.co/C-Tianyu/NanoJev) fine-tuned on [JS/TS Code Review Decisions](https://huggingface.co/datasets/yannelli/js-ts-code-review-decisions). On that dataset's test split it scores macro F1 0.452 and accuracy 0.831 over 15,600 decisions.
|
| 61 |
+
|
| 62 |
+
**Status: research baseline.** This checkpoint trained for half an epoch. It calls every clean snippet flawed and never flags `over_abstracted` or `duplicated_logic`. It is published for comparison; for code review use [d1-3B Code Review Decisions](https://huggingface.co/yannelli/d1-3b-code-review-decisions).
|
| 63 |
+
|
| 64 |
+
| | |
|
| 65 |
+
|---|---|
|
| 66 |
+
| Base model | [C-Tianyu/NanoJev](https://huggingface.co/C-Tianyu/NanoJev), Qwen3-0.6B with a choice head |
|
| 67 |
+
| Parameters | 596M (fp32, 2.4 GB) |
|
| 68 |
+
| Input | A state with `task`, `language` and `code`, and named questions |
|
| 69 |
+
| Output | P(yes) for yes/no questions; a distribution over options for choice and score questions |
|
| 70 |
+
| Training data | [JS/TS Code Review Decisions](https://huggingface.co/datasets/yannelli/js-ts-code-review-decisions), `decisions` config, `train` split (9,600 snippets) |
|
| 71 |
+
| Test results | Macro F1 0.452 (cutoff 0.5), accuracy 0.831, ECE 0.038 |
|
| 72 |
+
| License | Not specified (see License) |
|
| 73 |
+
|
| 74 |
+
## Questions
|
| 75 |
+
|
| 76 |
+
The model answers the 13 questions of the dataset's `decisions` config. They are the same for every snippet and are stored in the dataset's `manifest.json`. Each question's `criteria` carry the [flaw definition](https://huggingface.co/datasets/yannelli/js-ts-code-review-decisions#flaw-definitions) the labels follow.
|
| 77 |
+
|
| 78 |
+
| Question | Type | Asks | Answer |
|
| 79 |
+
|---|---|---|---|
|
| 80 |
+
| `unclear_naming` | noul | Does this code have unclear naming? | P(yes) |
|
| 81 |
+
| `noise_comments` | noul | Does this code have noise comments? | P(yes) |
|
| 82 |
+
| `dead_code` | noul | Does this code have dead code? | P(yes) |
|
| 83 |
+
| `inconsistent_style` | noul | Does this code have inconsistent style? | P(yes) |
|
| 84 |
+
| `over_abstracted` | noul | Does this code have over-abstraction? | P(yes) |
|
| 85 |
+
| `too_clever` | noul | Does this code have overly clever code? | P(yes) |
|
| 86 |
+
| `duplicated_logic` | noul | Does this code have duplicated logic? | P(yes) |
|
| 87 |
+
| `error_handling_problem` | noul | Does this code have an error handling problem? | P(yes) |
|
| 88 |
+
| `type_safety_hole` | noul | Does this code have a type safety hole? | P(yes) |
|
| 89 |
+
| `unsafe_input` | noul | Does this code have unsafe input handling? | P(yes) |
|
| 90 |
+
| `has_flaw` | noul | Does this code have at least one code review flaw? | P(yes) |
|
| 91 |
+
| `primary_flaw` | choice | Which code review problem matters most in this code? Choose none if the code is clean. | Distribution over `none` and the 10 flaws |
|
| 92 |
+
| `quality` | score | How mergeable is this code as written? | Distribution over 5 levels, 0 "Should not merge." to 4 "Exemplary. Merge as is." |
|
| 93 |
+
|
| 94 |
+
Quality level `i` is `quality` `i + 1` in the dataset's `default` config.
|
| 95 |
+
|
| 96 |
+
## Usage
|
| 97 |
+
|
| 98 |
+
```sh
|
| 99 |
+
pip install torch==2.14.1 transformers==5.19.0 huggingface_hub safetensors pyarrow
|
| 100 |
+
```
|
| 101 |
+
|
| 102 |
+
```python
|
| 103 |
+
import json
|
| 104 |
+
import os
|
| 105 |
+
import sys
|
| 106 |
+
from pathlib import Path
|
| 107 |
+
|
| 108 |
+
import torch
|
| 109 |
+
from huggingface_hub import hf_hub_download, snapshot_download
|
| 110 |
+
|
| 111 |
+
path = Path(snapshot_download("yannelli/nanojev-code-review-decisions"))
|
| 112 |
+
sys.path.insert(0, str(path / "training"))
|
| 113 |
+
os.environ.setdefault("NANOJEV_REPO", os.path.expanduser("~/.cache/nanojev")) # NanoJev's source goes here
|
| 114 |
+
from nanojev import STATE_CAP, answers_of, autocast, capped_state, examples_for, load_model
|
| 115 |
+
|
| 116 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 117 |
+
model, tokenizer, limit = load_model(path, device)
|
| 118 |
+
|
| 119 |
+
# The 13 questions the model was trained on, from the dataset's manifest.json
|
| 120 |
+
manifest = hf_hub_download("yannelli/js-ts-code-review-decisions", "manifest.json", repo_type="dataset")
|
| 121 |
+
questions = json.load(open(manifest))["questions"]
|
| 122 |
+
|
| 123 |
+
state = {
|
| 124 |
+
"task": "Define payroll bank account fields and decode a JSON account record, "
|
| 125 |
+
"accepting only a nine-digit routing number and a four-digit account suffix.",
|
| 126 |
+
"language": "TypeScript",
|
| 127 |
+
"code": """import { z } from 'zod';
|
| 128 |
+
|
| 129 |
+
const bankAccountSchema = z.object({
|
| 130 |
+
routingNumber: z.string().regex(/^\\d{9}$/),
|
| 131 |
+
accountSuffix: z.string().regex(/^\\d{4}$/)
|
| 132 |
+
});
|
| 133 |
+
export type PayrollBankAccount = z.infer<typeof bankAccountSchema>;
|
| 134 |
+
export function decodeBankAccount(serialized: string): PayrollBankAccount {
|
| 135 |
+
const account_payload = JSON.parse(serialized);
|
| 136 |
+
return account_payload;
|
| 137 |
+
}""",
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
case = {"id": "example", "state": state, "questions": questions}
|
| 141 |
+
text, _ = capped_state(case, tokenizer, STATE_CAP)
|
| 142 |
+
examples = examples_for([case], {"example": text}, tokenizer, limit, gold=False)
|
| 143 |
+
with torch.inference_mode(), autocast(device):
|
| 144 |
+
logits, _ = model(examples, tokenizer.pad_token_id)
|
| 145 |
+
answers = answers_of(case, examples, logits)
|
| 146 |
+
print(json.dumps(answers, indent=1))
|
| 147 |
+
```
|
| 148 |
+
|
| 149 |
+
The example is a test-split snippet the model did not see in training. Its gold labels are `inconsistent_style` (`account_payload` next to camelCase names) and `unsafe_input` (the `JSON.parse` result is returned without the schema defined above it). This model's answers, from the code above on CPU:
|
| 150 |
+
|
| 151 |
+
| Question | Gold | This model |
|
| 152 |
+
|---|---|---|
|
| 153 |
+
| `unclear_naming` | no | P(yes) 0.01 |
|
| 154 |
+
| `noise_comments` | no | P(yes) 0.00 |
|
| 155 |
+
| `dead_code` | no | P(yes) 0.16 |
|
| 156 |
+
| `inconsistent_style` | yes | P(yes) 0.11 |
|
| 157 |
+
| `over_abstracted` | no | P(yes) 0.09 |
|
| 158 |
+
| `too_clever` | no | P(yes) 0.09 |
|
| 159 |
+
| `duplicated_logic` | no | P(yes) 0.13 |
|
| 160 |
+
| `error_handling_problem` | no | P(yes) 0.01 |
|
| 161 |
+
| `type_safety_hole` | no | P(yes) 0.05 |
|
| 162 |
+
| `unsafe_input` | yes | P(yes) 0.13 |
|
| 163 |
+
| `has_flaw` | yes | P(yes) 0.58 |
|
| 164 |
+
| `primary_flaw` | `unsafe_input` | `none` (0.34) |
|
| 165 |
+
| `quality` | level 0 | level 4 (0.40); expected level 2.69 |
|
| 166 |
+
|
| 167 |
+
Answers use the dataset's answers format: a `noul` answer is P(yes), and `choice` and `score` answers carry `probabilities` over every option. One `{"id": ..., "answers": ...}` line per snippet scores directly with the dataset's `score_decisions.py`. The `quality` answer above, rounded:
|
| 168 |
+
|
| 169 |
+
```json
|
| 170 |
+
{
|
| 171 |
+
"type": "score",
|
| 172 |
+
"probabilities": {
|
| 173 |
+
"0": 0.04,
|
| 174 |
+
"1": 0.146,
|
| 175 |
+
"2": 0.3,
|
| 176 |
+
"3": 0.109,
|
| 177 |
+
"4": 0.404
|
| 178 |
+
}
|
| 179 |
+
}
|
| 180 |
+
```
|
| 181 |
+
|
| 182 |
+
The bundle loads with NanoJev's own code. `training/nanojev.py` fetches NanoJev's source from [C-Tianyu/NanoJev](https://huggingface.co/C-Tianyu/NanoJev) (revision `unified-games-v1`) into `NANOJEV_REPO` and refuses to run it unless the files match pinned SHA-256 hashes.
|
| 183 |
+
|
| 184 |
+
The model accepts other questions in the same schema, but it was trained and evaluated only on these 13. Answers to other questions are untested.
|
| 185 |
|
| 186 |
## Results
|
| 187 |
|
| 188 |
+
Results cover the `test` split of the `decisions` config: 300 tasks, 1,200 snippets, and 15,600 decisions, scored with the dataset's `score_decisions.py`. Each snippet was one request with all 13 questions. The four fine-tuned models used the same data and scorer.
|
|
|
|
|
|
|
| 189 |
|
| 190 |
+
| Model | Params | Macro F1 @ 0.5 | Accuracy | Log loss | Brier | ECE | Clean judged clean |
|
| 191 |
+
|---|---:|---:|---:|---:|---:|---:|---:|
|
| 192 |
+
| d1-3B | 3.1B | 0.917 | 0.940 | 0.172 | 0.072 | 0.017 | 0.950 |
|
| 193 |
+
| d1-omni-600M | 587M | 0.771 | 0.882 | 0.321 | 0.127 | 0.028 | 0.827 |
|
| 194 |
+
| GLiNER2.5-Decide | 486M | 0.687 | 0.863 | 0.398 | 0.149 | 0.039 | 0.683 |
|
| 195 |
+
| **NanoJev (this model)** | 596M | **0.452** | **0.831** | **0.459** | **0.173** | **0.038** | **0.000** |
|
| 196 |
+
| NanoJev base model, zero-shot | 596M | 0.237 | 0.192 | 1.413 | 0.559 | 0.510 | 0.000 |
|
| 197 |
+
| TF-IDF + logistic regression (dataset baseline) | - | 0.502 | 0.830 | 0.449 | 0.172 | 0.030 | 0.307 |
|
| 198 |
|
| 199 |
+
Higher is better for macro F1, accuracy, and clean judged clean; lower is better for log loss, Brier, and ECE. Clean judged clean is the share of the 300 clean test snippets with P(`has_flaw`) below 0.5. The dataset card's [Metrics](https://huggingface.co/datasets/yannelli/js-ts-code-review-decisions#metrics) section defines the other metrics.
|
| 200 |
|
| 201 |
+
95% intervals for this model, from 500 bootstrap resamples of test tasks: macro F1 0.431 to 0.472, accuracy 0.825 to 0.835, log loss 0.447 to 0.472, Brier 0.169 to 0.177. By question type its accuracy is 0.897 on the 11 nouls, 0.491 on `primary_flaw`, and 0.446 on `quality`.
|
| 202 |
|
| 203 |
+
### Per question
|
|
|
|
|
|
|
|
|
|
| 204 |
|
| 205 |
+
| Question | Type | Test positives | Accuracy | F1 @ 0.5 | Log loss | ECE |
|
| 206 |
+
|---|---|---:|---:|---:|---:|---:|
|
| 207 |
+
| `unclear_naming` | noul | 139 | 0.985 | 0.933 | 0.060 | 0.009 |
|
| 208 |
+
| `noise_comments` | noul | 178 | 0.998 | 0.992 | 0.012 | 0.001 |
|
| 209 |
+
| `dead_code` | noul | 223 | 0.858 | 0.414 | 0.403 | 0.030 |
|
| 210 |
+
| `inconsistent_style` | noul | 147 | 0.911 | 0.428 | 0.285 | 0.020 |
|
| 211 |
+
| `over_abstracted` | noul | 145 | 0.879 | 0.000 | 0.360 | 0.021 |
|
| 212 |
+
| `too_clever` | noul | 164 | 0.875 | 0.185 | 0.375 | 0.031 |
|
| 213 |
+
| `duplicated_logic` | noul | 171 | 0.858 | 0.000 | 0.325 | 0.147 |
|
| 214 |
+
| `error_handling_problem` | noul | 140 | 0.957 | 0.792 | 0.151 | 0.037 |
|
| 215 |
+
| `type_safety_hole` | noul | 148 | 0.921 | 0.527 | 0.281 | 0.029 |
|
| 216 |
+
| `unsafe_input` | noul | 165 | 0.872 | 0.252 | 0.371 | 0.039 |
|
| 217 |
+
| `has_flaw` | noul | 900 | 0.750 | 0.857 | 0.421 | 0.047 |
|
| 218 |
+
| `primary_flaw` | choice | | 0.491 | | 1.658 | 0.044 |
|
| 219 |
+
| `quality` | score | | 0.446 | | 1.265 | 0.039 |
|
| 220 |
+
|
| 221 |
+
### Calibration
|
| 222 |
+
|
| 223 |
+
Answers are the model's raw softmax (test ECE 0.038). `temperatures.json` holds one temperature per question, fitted on the `validation` split by log loss. Replacing each answer's distribution p with softmax(log p / T) moves test log loss from 0.459 to 0.456, Brier from 0.173 to 0.172 and ECE from 0.038 to 0.031. It never changes the top answer or the order of P(yes) values, so F1 and accuracy stay the same.
|
| 224 |
+
|
| 225 |
+
### Speed
|
| 226 |
+
|
| 227 |
+
Median wall-clock time for one snippet with all 13 questions, batch size 1. The CPU row covers 52 test snippets.
|
| 228 |
+
|
| 229 |
+
| Hardware | Runtime | p50 per snippet |
|
| 230 |
+
|---|---|---:|
|
| 231 |
+
| One H100 SXM | PyTorch, bf16 autocast | 115.4 ms |
|
| 232 |
+
| 16 Neoverse V3 CPU cores | PyTorch, fp32 | 18.2 s |
|
| 233 |
+
|
| 234 |
+
## Intended use
|
| 235 |
+
|
| 236 |
+
Research and comparison: a small decision model trained briefly on this data. It is not suited to flagging real code, because it calls every clean snippet flawed.
|
| 237 |
+
|
| 238 |
+
It is not a replacement for human review, a linter, a type checker, or a security audit.
|
| 239 |
|
| 240 |
## Training
|
| 241 |
|
| 242 |
+
- Data: the `train` split of the `decisions` config, 2,400 tasks and 9,600 snippets (124,800 decisions). The state is the task description, language and code. The training files match the SHA-256 hashes in the dataset's published `manifest.json`.
|
| 243 |
+
- The dataset keeps the 4 variants of a task in one split, so no test task was seen in training.
|
| 244 |
+
|
| 245 |
+
| Setting | Value |
|
| 246 |
+
|---|---|
|
| 247 |
+
| Method | Full fine-tune of the Qwen3-0.6B backbone (all 28 layers) and the choice head |
|
| 248 |
+
| Objective | Cross entropy over each question's candidate paths |
|
| 249 |
+
| Schedule | 1,200 steps of 4 snippets: half an epoch (4,800 of the 9,600 training snippets) |
|
| 250 |
+
| Optimizer | AdamW, learning rate 2e-5 (backbone) and 2e-4 (head), 100 warmup steps |
|
| 251 |
+
| Precision | bf16 autocast over fp32 weights |
|
| 252 |
+
| Hardware | One H100 SXM, 40 minutes |
|
| 253 |
+
| Validation loss | 0.453 at the end of training (300 validation snippets) |
|
| 254 |
+
|
| 255 |
+
`training/` holds the scripts that trained and scored this model (`nanojev.py`, `common.py`, `calibrate.py`). To rerun it:
|
| 256 |
+
|
| 257 |
+
```sh
|
| 258 |
+
pip install pyarrow numpy
|
| 259 |
+
hf download yannelli/js-ts-code-review-decisions --repo-type dataset --local-dir js-ts-code-review-decisions
|
| 260 |
+
export DECISIONS_DATA=js-ts-code-review-decisions
|
| 261 |
+
python training/nanojev.py train --ckpt ckpt --epochs 1 --max-steps 1200
|
| 262 |
+
python training/nanojev.py predict --ckpt ckpt --split test --out test-answers.jsonl
|
| 263 |
+
python js-ts-code-review-decisions/score_decisions.py test-answers.jsonl --split test
|
| 264 |
+
```
|
| 265 |
+
|
| 266 |
+
## Evaluation files
|
| 267 |
|
| 268 |
+
`eval/test-answers.jsonl` holds this model's answers to all 1,200 test snippets in the scorer's format, and `eval/score.txt` is the scorer's output for it. To rescore:
|
| 269 |
|
| 270 |
+
```sh
|
| 271 |
+
python js-ts-code-review-decisions/score_decisions.py eval/test-answers.jsonl --split test
|
| 272 |
+
```
|
| 273 |
+
|
| 274 |
+
## Limitations
|
| 275 |
+
|
| 276 |
+
- Gold labels come from blind model review and a model audit that accepted 98.3% of findings, not from human reviewers. Scores measure agreement with those labels.
|
| 277 |
+
- The snippets are synthetic, 5 to 80 lines long, and 25% of them are clean. False positive rates on real repositories will differ; measure them on code from your own projects before you set cutoffs.
|
| 278 |
+
- The model saw only JavaScript, JSX, TypeScript and TSX snippets with an English task description. Always pass a one-sentence `task`; inputs without one, whole files, diffs, and other languages are untested.
|
| 279 |
+
- Weakest flags by test F1: `over_abstracted` (0.00), `duplicated_logic` (0.00), `too_clever` (0.18). `quality` is the hardest question (accuracy 0.446); 46% of its wrong answers are one level off.
|
| 280 |
+
- `unsafe_input` follows the dataset's definition (external data used unchecked). It is not a vulnerability scanner and does not replace a security review.
|
| 281 |
+
- `has_flaw` P(yes) stays above 0.54 on every test snippet, so all 300 clean snippets are called flawed.
|
| 282 |
+
- `over_abstracted` and `duplicated_logic` have F1 0: their highest test P(yes) is 0.47 and 0.43.
|
| 283 |
+
|
| 284 |
+
## Other models trained on this dataset
|
| 285 |
+
|
| 286 |
+
| Model | Base | Params | Macro F1 @ 0.5 | Accuracy |
|
| 287 |
+
|---|---|---:|---:|---:|
|
| 288 |
+
| [d1-3B](https://huggingface.co/yannelli/d1-3b-code-review-decisions) | LFM2.5-VL-3B with Liquid AI's d1 decision readout | 3.1B | 0.917 | 0.940 |
|
| 289 |
+
| [d1-omni-600M](https://huggingface.co/yannelli/d1-omni-600m-code-review-decisions) | LFM2.5 encoder with a decision head | 587M | 0.771 | 0.882 |
|
| 290 |
+
| [GLiNER2.5-Decide](https://huggingface.co/yannelli/gliner2.5-decide-code-review-decisions) | GLiNER2 DeBERTa-v3 encoder with a classification head | 486M | 0.687 | 0.863 |
|
| 291 |
+
| [NanoJev](https://huggingface.co/yannelli/nanojev-code-review-decisions) (this model) | Qwen3-0.6B with a choice head | 596M | 0.452 | 0.831 |
|
| 292 |
+
|
| 293 |
+
[yannelli/code-review-decisions-GGUF](https://huggingface.co/yannelli/code-review-decisions-GGUF) has GGUF builds of the two Liquid models for llama.cpp and Ollama.
|
| 294 |
|
| 295 |
## License
|
| 296 |
|
| 297 |
+
Not specified. The NanoJev source code is MIT licensed and the Qwen3-0.6B base model is Apache 2.0, but the [C-Tianyu/NanoJev](https://huggingface.co/C-Tianyu/NanoJev) release does not state a license for its weights, and these weights derive from them.
|
| 298 |
+
|
| 299 |
+
The training data, [JS/TS Code Review Decisions](https://huggingface.co/datasets/yannelli/js-ts-code-review-decisions), is CC BY 4.0.
|
| 300 |
+
|
| 301 |
+
## Citation
|
| 302 |
+
|
| 303 |
+
```bibtex
|
| 304 |
+
@misc{yannelli2026nanojevcodereview,
|
| 305 |
+
title = {NanoJev Code Review Decisions},
|
| 306 |
+
author = {Ryan Yannelli},
|
| 307 |
+
year = {2026},
|
| 308 |
+
publisher = {Hugging Face},
|
| 309 |
+
howpublished = {\url{https://huggingface.co/yannelli/nanojev-code-review-decisions}}
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
@misc{yannelli2026jstscodereviewdecisions,
|
| 313 |
+
title = {JS/TS Code Review Decisions},
|
| 314 |
+
author = {Ryan Yannelli},
|
| 315 |
+
year = {2026},
|
| 316 |
+
publisher = {Hugging Face},
|
| 317 |
+
howpublished = {\url{https://huggingface.co/datasets/yannelli/js-ts-code-review-decisions}}
|
| 318 |
+
}
|
| 319 |
+
```
|
| 320 |
+
|
| 321 |
+
## Maintainer
|
| 322 |
+
|
| 323 |
+
[Ryan Yannelli](https://ryanyannelli.com) | 10/12/2026 | [GitHub: @yannelli](https://github.com/yannelli) | [Hugging Face: yannelli](https://huggingface.co/yannelli)
|
eval/score.txt
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
scored 1200/1200 cases, 15,600 decisions; off-menu mass 0.0000; missing mass 0.0000
|
| 2 |
+
|
| 3 |
+
| Macro F1 at 0.5 | Accuracy | Log loss | Brier | ECE | p50 latency |
|
| 4 |
+
|---:|---:|---:|---:|---:|---:|
|
| 5 |
+
| 0.452 (0.431 to 0.472) | 0.831 (0.825 to 0.835) | 0.459 (0.447 to 0.472) | 0.173 (0.169 to 0.177) | 0.038 | 115.4 ms |
|
| 6 |
+
|
| 7 |
+
Accuracy by type: noul 0.897, choice 0.491, score 0.446; clean cases judged clean 0.000
|
| 8 |
+
|
| 9 |
+
| Question | Type | Positives | Accuracy | F1 at 0.5 | Log loss | Brier | ECE |
|
| 10 |
+
|---|---|---:|---:|---:|---:|---:|---:|
|
| 11 |
+
| `unclear_naming` | noul | 139 | 0.985 | 0.933 | 0.060 | 0.013 | 0.009 |
|
| 12 |
+
| `noise_comments` | noul | 178 | 0.998 | 0.992 | 0.012 | 0.002 | 0.001 |
|
| 13 |
+
| `dead_code` | noul | 223 | 0.858 | 0.414 | 0.403 | 0.120 | 0.030 |
|
| 14 |
+
| `inconsistent_style` | noul | 147 | 0.911 | 0.428 | 0.285 | 0.078 | 0.020 |
|
| 15 |
+
| `over_abstracted` | noul | 145 | 0.879 | 0.000 | 0.360 | 0.104 | 0.021 |
|
| 16 |
+
| `too_clever` | noul | 164 | 0.875 | 0.185 | 0.375 | 0.108 | 0.031 |
|
| 17 |
+
| `duplicated_logic` | noul | 171 | 0.858 | 0.000 | 0.325 | 0.094 | 0.147 |
|
| 18 |
+
| `error_handling_problem` | noul | 140 | 0.957 | 0.792 | 0.151 | 0.037 | 0.037 |
|
| 19 |
+
| `type_safety_hole` | noul | 148 | 0.921 | 0.527 | 0.281 | 0.073 | 0.029 |
|
| 20 |
+
| `unsafe_input` | noul | 165 | 0.872 | 0.252 | 0.371 | 0.108 | 0.039 |
|
| 21 |
+
| `has_flaw` | noul | 900 | 0.750 | 0.857 | 0.421 | 0.145 | 0.047 |
|
| 22 |
+
| `primary_flaw` | choice | | 0.491 | | 1.658 | 0.692 | 0.044 |
|
| 23 |
+
| `quality` | score | | 0.446 | | 1.265 | 0.668 | 0.039 |
|
eval/test-answers.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
temperatures.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"fitted_on": "yannelli/js-ts-code-review-decisions, decisions config, validation split",
|
| 3 |
+
"apply": "p -> softmax(log(p) / T), per question",
|
| 4 |
+
"temperatures": {
|
| 5 |
+
"unclear_naming": 1.016,
|
| 6 |
+
"noise_comments": 1.126,
|
| 7 |
+
"dead_code": 0.956,
|
| 8 |
+
"inconsistent_style": 0.981,
|
| 9 |
+
"over_abstracted": 1.03,
|
| 10 |
+
"too_clever": 1.056,
|
| 11 |
+
"duplicated_logic": 0.661,
|
| 12 |
+
"error_handling_problem": 1.031,
|
| 13 |
+
"type_safety_hole": 1.065,
|
| 14 |
+
"unsafe_input": 0.93,
|
| 15 |
+
"has_flaw": 0.97,
|
| 16 |
+
"primary_flaw": 1.002,
|
| 17 |
+
"quality": 0.804
|
| 18 |
+
}
|
| 19 |
+
}
|