maka/docs/eval/terminal-bench-2.1-deepseek-v4-flash-nine-arm.csv

1	task	codex	maka	pi	dsh	zcode	reasonix	opencode	kimi-code	claude-code
2	adaptive-rejection-sampler	pass	pass	pass	fail	fail	fail	pass	fail	fail
3	bn-fit-modify	fail	fail	pass	pass	pass	pass	fail	fail	pass
4	break-filter-js-from-html	pass	pass	pass	pass	pass	pass	pass	pass	pass
5	build-cython-ext	pass	pass	pass	pass	pass	pass	pass	pass	fail
6	build-pmars	pass	pass	pass	pass	pass	pass	pass	pass	fail
7	build-pov-ray	pass	pass	pass	pass	fail	pass	fail	pass	fail
8	caffe-cifar-10	pass	fail	pass	fail	pass	fail	pass	fail	fail
9	cancel-async-tasks	fail	pass	fail	pass	pass	pass	fail	fail	pass
10	chess-best-move	fail	fail	fail	pass	pass	fail	fail	fail	fail
11	circuit-fibsqrt	pass	pass	pass	pass	pass	fail	pass	pass	pass
12	cobol-modernization	pass	pass	pass	pass	pass	pass	pass	pass	pass
13	code-from-image	pass	pass	pass	pass	pass	pass	fail	fail	fail
14	compile-compcert	pass	pass	pass	pass	pass	pass	fail	fail	pass
15	configure-git-webserver	pass	pass	pass	pass	pass	fail	pass	fail	pass
16	constraints-scheduling	pass	pass	pass	pass	pass	pass	pass	pass	pass
17	count-dataset-tokens	pass	pass	pass	pass	pass	pass	pass	pass	pass
18	crack-7z-hash	pass	pass	pass	pass	pass	pass	pass	fail	fail
19	custom-memory-heap-crash	pass	pass	pass	pass	pass	pass	pass	pass	pass
20	db-wal-recovery	pass	fail	pass	fail	pass	pass	pass	fail	pass
21	distribution-search	pass	pass	pass	pass	pass	pass	pass	pass	pass
22	dna-assembly	pass	pass	fail	fail	fail	fail	fail	fail	fail
23	dna-insert	pass	fail	pass	pass	fail	fail	fail	fail	fail
24	extract-elf	pass	fail	fail	fail	pass	fail	fail	fail	pass
25	extract-moves-from-video	fail	fail	fail	fail	fail	fail	fail	fail	fail
26	feal-differential-cryptanalysis	pass	pass	pass	pass	pass	pass	pass	pass	pass
27	feal-linear-cryptanalysis	pass	pass	pass	pass	pass	fail	pass	pass	pass
28	filter-js-from-html	fail	fail	fail	fail	fail	fail	fail	fail	fail
29	financial-document-processor	pass	pass	pass	pass	pass	pass	pass	fail	pass
30	fix-code-vulnerability	pass	pass	pass	pass	pass	pass	pass	pass	pass
31	fix-git	pass	pass	pass	pass	pass	pass	pass	pass	pass
32	fix-ocaml-gc	pass	pass	pass	pass	pass	pass	pass	pass	pass
33	gcode-to-text	fail	fail	fail	fail	fail	fail	fail	fail	fail
34	git-leak-recovery	pass	pass	pass	pass	pass	pass	pass	pass	fail
35	git-multibranch	pass	pass	pass	pass	pass	pass	pass	pass	pass
36	gpt2-codegolf	pass	fail	fail	fail	fail	fail	fail	fail	fail
37	headless-terminal	pass	pass	pass	pass	pass	pass	pass	pass	pass
38	hf-model-inference	pass	pass	pass	pass	pass	pass	pass	pass	pass
39	install-windows-3.11	pass	pass	fail	fail	fail	fail	fail	fail	fail
40	kv-store-grpc	pass	pass	pass	fail	fail	fail	pass	pass	fail
41	large-scale-text-editing	pass	pass	pass	pass	pass	pass	pass	pass	pass
42	largest-eigenval	pass	pass	fail	pass	fail	fail	pass	fail	fail
43	llm-inference-batching-scheduler	pass	pass	pass	pass	pass	pass	pass	pass	pass
44	log-summary-date-ranges	pass	pass	pass	pass	pass	pass	pass	pass	pass
45	mailman	pass	pass	pass	fail	pass	pass	fail	pass	fail
46	make-doom-for-mips	fail	fail	fail	fail	fail	fail	fail	fail	fail
47	make-mips-interpreter	pass	fail	fail	pass	fail	fail	fail	fail	fail
48	mcmc-sampling-stan	pass	pass	pass	pass	pass	pass	pass	fail	fail
49	merge-diff-arc-agi-task	pass	pass	pass	pass	pass	pass	pass	pass	fail
50	model-extraction-relu-logits	fail	pass	fail	pass	pass	pass	fail	pass	fail
51	modernize-scientific-stack	pass	pass	pass	pass	pass	pass	pass	pass	pass
52	mteb-leaderboard	pass	pass	fail	pass	pass	pass	pass	pass	pass
53	mteb-retrieve	pass	pass	pass	pass	pass	pass	fail	fail	pass
54	multi-source-data-merger	pass	pass	pass	pass	pass	pass	pass	pass	pass
55	nginx-request-logging	pass	pass	pass	pass	pass	pass	pass	pass	fail
56	openssl-selfsigned-cert	pass	pass	pass	pass	pass	pass	pass	pass	pass
57	overfull-hbox	pass	fail	pass	pass	pass	fail	pass	fail	pass
58	password-recovery	pass	pass	pass	fail	pass	fail	pass	pass	pass
59	path-tracing	pass	pass	pass	pass	pass	pass	pass	fail	fail
60	path-tracing-reverse	pass	pass	pass	pass	fail	fail	fail	fail	pass
61	polyglot-c-py	pass	pass	pass	pass	pass	pass	pass	pass	pass
62	polyglot-rust-c	pass	pass	pass	pass	pass	pass	pass	pass	pass
63	portfolio-optimization	pass	pass	pass	pass	pass	pass	pass	pass	pass
64	protein-assembly	fail	fail	fail	fail	fail	pass	fail	fail	pass
65	prove-plus-comm	pass	pass	pass	pass	pass	pass	pass	pass	pass
66	pypi-server	fail	pass	pass	fail	fail	pass	pass	pass	pass
67	pytorch-model-cli	pass	pass	pass	pass	pass	pass	pass	fail	fail
68	pytorch-model-recovery	pass	pass	fail	fail	pass	pass	fail	pass	pass
69	qemu-alpine-ssh	fail	pass	fail	fail	fail	pass	pass	pass	pass
70	qemu-startup	pass	pass	pass	fail	fail	fail	pass	pass	pass
71	query-optimize	pass	pass	fail	pass	fail	pass	pass	pass	fail
72	raman-fitting	fail	fail	fail	fail	pass	fail	fail	fail	fail
73	regex-chess	pass	pass	pass	pass	fail	fail	fail	fail	fail
74	regex-log	pass	pass	pass	pass	pass	pass	pass	pass	pass
75	reshard-c4-data	pass	pass	pass	pass	pass	pass	pass	pass	pass
76	rstan-to-pystan	pass	pass	pass	fail	pass	pass	fail	pass	fail
77	sam-cell-seg	pass	pass	pass	pass	pass	fail	pass	fail	fail
78	sanitize-git-repo	fail	pass	pass	pass	fail	pass	fail	pass	fail
79	schemelike-metacircular-eval	pass	pass	pass	pass	pass	pass	pass	pass	fail
80	sparql-university	pass	pass	pass	pass	pass	pass	pass	pass	pass
81	sqlite-db-truncate	pass	pass	pass	pass	pass	pass	pass	pass	pass
82	sqlite-with-gcov	pass	pass	pass	pass	pass	pass	pass	pass	fail
83	torch-pipeline-parallelism	fail	fail	fail	fail	fail	fail	fail	fail	fail
84	torch-tensor-parallelism	pass	fail	pass	pass	fail	fail	fail	pass	pass
85	train-fasttext	fail	fail	fail	pass	fail	fail	fail	fail	fail
86	tune-mjcf	pass	pass	fail	pass	fail	pass	fail	fail	fail
87	video-processing	fail	fail	fail	fail	fail	fail	pass	fail	fail
88	vulnerable-secret	pass	pass	pass	pass	pass	pass	pass	pass	pass
89	winning-avg-corewars	pass	pass	pass	pass	pass	pass	fail	pass	pass
90	write-compressor	pass	fail	pass	fail	pass	pass	pass	pass	fail