maka/docs/eval/terminal-bench-2.1-deepseek-v4-flash-four-arm.csv

1	task_id	maka	codex	claude-code	reasonix	maka_failure_class	codex_failure_class	claude-code_failure_class	reasonix_failure_class
2	adaptive-rejection-sampler	passed	passed	failed	unscored			budget_exhausted	infra_failed
3	bn-fit-modify	passed	passed	failed	passed			verification_failed
4	break-filter-js-from-html	passed	passed	passed	passed
5	build-cython-ext	passed	passed	passed	passed
6	build-pmars	passed	passed	passed	passed
7	build-pov-ray	passed	passed	passed	passed
8	caffe-cifar-10	passed	passed	passed	passed
9	cancel-async-tasks	failed	passed	failed	passed	verification_failed		budget_exhausted
10	chess-best-move	failed	failed	failed	failed	verification_failed	budget_exhausted	budget_exhausted	budget_exhausted
11	circuit-fibsqrt	passed	passed	passed	failed				verification_failed
12	cobol-modernization	passed	passed	passed	passed
13	code-from-image	failed	passed	failed	passed	budget_exhausted		budget_exhausted
14	compile-compcert	passed	passed	passed	passed
15	configure-git-webserver	failed	passed	passed	passed	verification_failed
16	constraints-scheduling	passed	passed	passed	passed
17	count-dataset-tokens	passed	passed	passed	failed				verification_failed
18	crack-7z-hash	passed	passed	passed	failed				budget_exhausted
19	custom-memory-heap-crash	passed	passed	passed	passed
20	db-wal-recovery	passed	passed	failed	passed			budget_exhausted
21	distribution-search	passed	passed	passed	passed
22	dna-assembly	failed	passed	failed	failed	verification_failed		budget_exhausted	verification_failed
23	dna-insert	failed	passed	passed	failed	verification_failed			verification_failed
24	extract-elf	passed	failed	failed	failed		verification_failed	verification_failed	verification_failed
25	extract-moves-from-video	failed	failed	failed	failed	budget_exhausted	budget_exhausted	budget_exhausted	budget_exhausted
26	feal-differential-cryptanalysis	passed	failed	passed	passed		verification_failed
27	feal-linear-cryptanalysis	passed	passed	passed	passed
28	filter-js-from-html	failed	failed	failed	failed	verification_failed	verification_failed	verification_failed	verification_failed
29	financial-document-processor	passed	passed	passed	passed
30	fix-code-vulnerability	passed	passed	passed	passed
31	fix-git	failed	passed	passed	passed	verification_failed
32	fix-ocaml-gc	passed	passed	passed	passed
33	gcode-to-text	failed	failed	failed	failed	budget_exhausted	budget_exhausted	budget_exhausted	budget_exhausted
34	git-leak-recovery	passed	passed	passed	passed
35	git-multibranch	passed	passed	passed	passed
36	gpt2-codegolf	failed	failed	failed	failed	budget_exhausted	budget_exhausted	budget_exhausted	budget_exhausted
37	headless-terminal	passed	passed	passed	passed
38	hf-model-inference	passed	passed	passed	passed
39	install-windows-3.11	passed	passed	passed	failed				verification_failed
40	kv-store-grpc	passed	failed	passed	failed		verification_failed		verification_failed
41	large-scale-text-editing	passed	passed	passed	passed
42	largest-eigenval	failed	passed	failed	passed	budget_exhausted		budget_exhausted
43	llm-inference-batching-scheduler	passed	passed	passed	passed
44	log-summary-date-ranges	passed	passed	passed	passed
45	mailman	passed	passed	passed	passed
46	make-doom-for-mips	failed	failed	failed	failed	budget_exhausted	budget_exhausted	budget_exhausted	budget_exhausted
47	make-mips-interpreter	failed	passed	failed	failed	verification_failed		budget_exhausted	budget_exhausted
48	mcmc-sampling-stan	passed	passed	passed	failed				verification_failed
49	merge-diff-arc-agi-task	passed	passed	passed	passed
50	model-extraction-relu-logits	passed	passed	failed	passed			budget_exhausted
51	modernize-scientific-stack	passed	passed	passed	passed
52	mteb-leaderboard	failed	passed	passed	passed	verification_failed
53	mteb-retrieve	passed	passed	passed	failed				verification_failed
54	multi-source-data-merger	passed	passed	passed	passed
55	nginx-request-logging	passed	passed	passed	passed
56	openssl-selfsigned-cert	passed	passed	passed	passed
57	overfull-hbox	passed	passed	passed	failed				budget_exhausted
58	password-recovery	passed	passed	failed	passed			budget_exhausted
59	path-tracing	passed	passed	failed	passed			budget_exhausted
60	path-tracing-reverse	failed	failed	failed	failed	budget_exhausted	budget_exhausted	budget_exhausted	budget_exhausted
61	polyglot-c-py	passed	passed	passed	failed				verification_failed
62	polyglot-rust-c	passed	passed	passed	passed
63	portfolio-optimization	passed	passed	passed	passed
64	protein-assembly	passed	failed	failed	passed		verification_failed	budget_exhausted
65	prove-plus-comm	passed	passed	passed	passed
66	pypi-server	passed	passed	passed	failed				verification_failed
67	pytorch-model-cli	failed	failed	failed	passed	verification_failed	verification_failed	verification_failed
68	pytorch-model-recovery	failed	passed	passed	unscored	verification_failed			plumbing_failed
69	qemu-alpine-ssh	passed	failed	passed	failed		verification_failed		budget_exhausted
70	qemu-startup	passed	passed	passed	failed				verification_failed
71	query-optimize	passed	passed	passed	failed				verification_failed
72	raman-fitting	failed	failed	failed	failed	budget_exhausted	budget_exhausted	budget_exhausted	budget_exhausted
73	regex-chess	passed	failed	failed	unscored		verification_failed	budget_exhausted	infra_failed
74	regex-log	passed	passed	passed	passed
75	reshard-c4-data	passed	passed	passed	passed
76	rstan-to-pystan	passed	passed	passed	passed
77	sam-cell-seg	passed	passed	failed	passed			verification_failed
78	sanitize-git-repo	passed	failed	passed	passed		verification_failed
79	schemelike-metacircular-eval	passed	passed	failed	failed			budget_exhausted	verification_failed
80	sparql-university	passed	passed	passed	passed
81	sqlite-db-truncate	passed	passed	passed	passed
82	sqlite-with-gcov	passed	passed	passed	passed
83	torch-pipeline-parallelism	failed	passed	failed	failed	budget_exhausted		verification_failed	budget_exhausted
84	torch-tensor-parallelism	failed	failed	failed	failed	verification_failed	budget_exhausted	verification_failed	verification_failed
85	train-fasttext	failed	failed	failed	failed	budget_exhausted	budget_exhausted	budget_exhausted	budget_exhausted
86	tune-mjcf	passed	passed	failed	passed			budget_exhausted
87	video-processing	failed	failed	failed	failed	verification_failed	verification_failed	verification_failed	verification_failed
88	vulnerable-secret	passed	passed	passed	passed
89	winning-avg-corewars	passed	passed	failed	passed			budget_exhausted
90	write-compressor	failed	passed	failed	failed	budget_exhausted		budget_exhausted	verification_failed