maka/docs/eval/terminal-bench-2.1-deepseek-v4-flash-maka-vs-opencode.csv

1	task_id	maka	opencode	maka_failure_class	opencode_failure_class
2	adaptive-rejection-sampler	failed	failed	budget_exhausted	budget_exhausted
3	bn-fit-modify	passed	failed		verification_failed
4	break-filter-js-from-html	passed	passed
5	build-cython-ext	passed	passed
6	build-pmars	passed	passed
7	build-pov-ray	passed	passed
8	caffe-cifar-10	passed	failed		budget_exhausted
9	cancel-async-tasks	failed	failed	verification_failed	verification_failed
10	chess-best-move	failed	failed	verification_failed	budget_exhausted
11	circuit-fibsqrt	passed	passed
12	cobol-modernization	passed	passed
13	code-from-image	passed	failed		budget_exhausted
14	compile-compcert	failed	passed	budget_exhausted
15	configure-git-webserver	passed	passed
16	constraints-scheduling	passed	passed
17	count-dataset-tokens	passed	passed
18	crack-7z-hash	passed	passed
19	custom-memory-heap-crash	passed	passed
20	db-wal-recovery	passed	failed		budget_exhausted
21	distribution-search	passed	passed
22	dna-assembly	failed	failed	verification_failed	verification_failed
23	dna-insert	failed	passed	verification_failed
24	extract-elf	failed	failed	verification_failed	verification_failed
25	extract-moves-from-video	failed	failed	budget_exhausted	budget_exhausted
26	feal-differential-cryptanalysis	passed	passed
27	feal-linear-cryptanalysis	passed	passed
28	filter-js-from-html	failed	failed	verification_failed	verification_failed
29	financial-document-processor	passed	passed
30	fix-code-vulnerability	passed	passed
31	fix-git	passed	failed		verification_failed
32	fix-ocaml-gc	passed	passed
33	gcode-to-text	failed	failed	budget_exhausted	budget_exhausted
34	git-leak-recovery	passed	passed
35	git-multibranch	passed	passed
36	gpt2-codegolf	failed	failed	budget_exhausted	budget_exhausted
37	headless-terminal	passed	passed
38	hf-model-inference	passed	passed
39	install-windows-3.11	failed	failed	budget_exhausted	budget_exhausted
40	kv-store-grpc	passed	passed
41	large-scale-text-editing	passed	passed
42	largest-eigenval	failed	failed	budget_exhausted	budget_exhausted
43	llm-inference-batching-scheduler	passed	passed
44	log-summary-date-ranges	passed	passed
45	mailman	passed	passed
46	make-doom-for-mips	failed	failed	budget_exhausted	budget_exhausted
47	make-mips-interpreter	failed	failed	budget_exhausted	budget_exhausted
48	mcmc-sampling-stan	passed	passed
49	merge-diff-arc-agi-task	passed	passed
50	model-extraction-relu-logits	passed	failed		budget_exhausted
51	modernize-scientific-stack	passed	passed
52	mteb-leaderboard	failed	failed	verification_failed	budget_exhausted
53	mteb-retrieve	passed	failed		verification_failed
54	multi-source-data-merger	passed	passed
55	nginx-request-logging	passed	passed
56	openssl-selfsigned-cert	passed	passed
57	overfull-hbox	failed	failed	verification_failed	verification_failed
58	password-recovery	failed	passed	budget_exhausted
59	path-tracing	passed	failed		budget_exhausted
60	path-tracing-reverse	failed	failed	budget_exhausted	budget_exhausted
61	polyglot-c-py	passed	passed
62	polyglot-rust-c	passed	passed
63	portfolio-optimization	passed	passed
64	protein-assembly	failed	failed	verification_failed	verification_failed
65	prove-plus-comm	passed	passed
66	pypi-server	passed	passed
67	pytorch-model-cli	passed	passed
68	pytorch-model-recovery	passed	passed
69	qemu-alpine-ssh	failed	failed	budget_exhausted	budget_exhausted
70	qemu-startup	failed	passed	budget_exhausted
71	query-optimize	passed	passed
72	raman-fitting	failed	failed	verification_failed	budget_exhausted
73	regex-chess	passed	failed		budget_exhausted
74	regex-log	passed	passed
75	reshard-c4-data	passed	passed
76	rstan-to-pystan	failed	failed	budget_exhausted	runtime_error
77	sam-cell-seg	passed	passed
78	sanitize-git-repo	passed	failed		verification_failed
79	schemelike-metacircular-eval	passed	failed		verification_failed
80	sparql-university	passed	failed		verification_failed
81	sqlite-db-truncate	passed	passed
82	sqlite-with-gcov	passed	failed		budget_exhausted
83	torch-pipeline-parallelism	failed	failed	verification_failed	budget_exhausted
84	torch-tensor-parallelism	failed	failed	verification_failed	verification_failed
85	train-fasttext	failed	failed	budget_exhausted	budget_exhausted
86	tune-mjcf	passed	failed		budget_exhausted
87	video-processing	failed	failed	verification_failed	verification_failed
88	vulnerable-secret	passed	passed
89	winning-avg-corewars	passed	failed		verification_failed
90	write-compressor	passed	failed		budget_exhausted